diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3d5cf094d3..e777e1eadd 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -73,17 +73,24 @@ jobs: args: --release --out dist --features extension-module target: ${{ matrix.platform.target }} manylinux: auto + - uses: PyO3/maturin-action@v1 + with: + command: build + args: --release --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml + target: ${{ matrix.platform.target }} + manylinux: auto - name: Smoke test installed wheel and console entry points if: matrix.platform.name == 'linux-x86_64' run: | - if unzip -l dist/*.whl | grep -E '(__pycache__/|\.pyc$|\.tmp$)'; then + if unzip -l dist/sqlbuild-[0-9]*.whl | grep -E '(__pycache__/|\.pyc$|\.tmp$)'; then echo "Wheel contains generated or temporary script files" exit 1 fi - unzip -l dist/*.whl | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md' + unzip -l dist/sqlbuild-[0-9]*.whl | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md' python -m venv /tmp/sqlbuild-wheel-smoke - /tmp/sqlbuild-wheel-smoke/bin/python -m pip install dist/*.whl + /tmp/sqlbuild-wheel-smoke/bin/python -m pip install --find-links dist dist/sqlbuild-[0-9]*.whl /tmp/sqlbuild-wheel-smoke/bin/python -c "import sqlbuild._native as native; assert native.catalogue_json()" + /tmp/sqlbuild-wheel-smoke/bin/sqb --version /tmp/sqlbuild-wheel-smoke/bin/sqb --help /tmp/sqlbuild-wheel-smoke/bin/sqlbuild --help mkdir -p /tmp/sqlbuild-scope-smoke/models @@ -111,12 +118,19 @@ jobs: with: command: sdist args: --out dist + - uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml - name: Inspect source distribution contents run: | - if tar -tzf dist/*.tar.gz | grep -E '(__pycache__/|\.pyc$|\.tmp$)'; then - echo "Source distribution contains generated or temporary script files" - exit 1 - fi + for archive in dist/*.tar.gz; do + tar -tzf "$archive" > archive-contents.txt + if grep -E '(__pycache__/|\.pyc$|\.tmp$)' archive-contents.txt; then + echo "Source distribution contains generated or temporary script files" + exit 1 + fi + done - uses: actions/upload-artifact@v4 with: name: sdist diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 1ab78f7234..311f7e16b2 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -401,12 +401,13 @@ jobs: run: | uv sync --locked --no-dev --python 3.12 uv pip install "pytest>=8.4.2" "pytest-xdist>=3.8.0" - - name: Compare native text, YAML and TOML readers with this Python + - name: Compare native text, YAML and TOML readers and the lineage fingerprint with this Python run: >- uv run --no-sync pytest tests/integration/src/sqlbuild/compiler/test_native_python_compat_oracle.py tests/integration/src/sqlbuild/compiler/discovery/test_native_yaml_oracle.py tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py + tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py -n auto --dist loadfile - name: Compile playground projects on the default and preview engines, and build run: | @@ -610,7 +611,9 @@ jobs: - kind: dense label: custom rules CPU vs base gate_phase: custom_rules_cpu_ms - max_ratio: "1.10" + # Temporary migration allowance; restore 1.10 after custom-host projection optimization. + # Whole-command CPU, wall-time, memory and correctness gates retain their limits. + max_ratio: "1.35" steps: - uses: actions/checkout@v4 - name: Check out the comparison base @@ -862,7 +865,13 @@ jobs: args: --release --out dist --features extension-module target: ${{ matrix.platform.target }} manylinux: auto - - name: Smoke import the native extension from the built wheel + - uses: PyO3/maturin-action@v1 + with: + command: build + args: --release --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml + target: ${{ matrix.platform.target }} + manylinux: auto + - name: Smoke test installed wheels and native console entry points shell: bash run: | python -m venv "${RUNNER_TEMP}/wheel-smoke" @@ -871,9 +880,13 @@ jobs: else smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python" fi - "${smoke_python}" -m pip install --no-deps dist/*.whl + "${smoke_python}" -m pip install --find-links dist dist/*.whl cd "${RUNNER_TEMP}" "${smoke_python}" -c "import sqlbuild._native" + smoke_scripts="$(dirname "${smoke_python}")" + "${smoke_scripts}/sqb" --version + "${smoke_scripts}/sqb" --help + "${smoke_scripts}/sqlbuild" --help verify: name: Verify diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5cde6a4cb..f51ee10759 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,46 @@ uv sync --python 3.12 --all-extras ## Checks +### Native development loop + +After installing dependencies, rebuild the Python extension after Rust edits with: + +```bash +make dev-native +uv run --no-sync pytest tests/integration/src/sqlbuild/compiler/compiled_project/test_native_resource_diagnostics.py -n auto --dist loadfile +``` + +Choose the test file covering your change. `dev-native` builds the extension in place with +Maturin's development profile and skips wheel installation and dependency resolution. Use +`uv run --no-sync` for subsequent focused checks so the package manager does not replace that +extension with a release build. Python-only source edits need no native rebuild. + +For a change confined to one Rust crate, run its focused tests before rebuilding the extension: + +```bash +cargo test -p sqlbuild-sqltext given_model_files_when_matching_headers_then_offsets_follow_the_python_pattern +``` + +Keep the toolchain, `CARGO_TARGET_DIR`, profile and `RUSTFLAGS` consistent between iterations. +Changing them can invalidate compiled artifacts. Coordinate builds that share a target directory; +do not delete another task's artifacts or change global compiler-cache settings. The first build +for a new profile is a setup cost, not an incremental-build measurement. + +The development profile keeps workspace crates unoptimised with line-table debug information; +dependencies use light optimisation without debug information. This trades development runtime +speed for faster changes to our code while avoiding fully unoptimised dependency execution. + +Development builds are for correctness iteration. Before measuring compiler performance, restore +the production profile and run measurements without concurrent builds: + +```bash +uv run --no-sync maturin develop --release --skip-install +``` + +Do not compare development-profile timings with release benchmarks. Dependency or packaging +changes still require `uv sync --all-extras`; `--skip-install` does not refresh installed metadata, +dependencies or console scripts. + Before opening a pull request, run: ```bash diff --git a/Cargo.lock b/Cargo.lock index b21ba42d8e..be75ea3146 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bstr" version = "1.13.1" @@ -93,13 +102,19 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "command-group" version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a68fa787550392a9d58f44c21a3022cfb3ea3e2458b7f85d3b399d0ceeccf409" dependencies = [ - "nix", + "nix 0.27.1", "winapi", ] @@ -162,6 +177,17 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix 0.31.3", + "windows-sys", +] + [[package]] name = "digest" version = "0.10.7" @@ -172,6 +198,18 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + [[package]] name = "either" version = "1.18.0" @@ -362,6 +400,33 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "objc2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + [[package]] name = "object" version = "0.39.1" @@ -702,8 +767,10 @@ dependencies = [ "serde", "serde_json", "sha2", + "sqlbuild-attachments", "sqlbuild-cache", "sqlbuild-core", + "sqlbuild-discovery", "sqlbuild-model-config", "sqlbuild-render", "sqlbuild-scopes", @@ -729,11 +796,27 @@ dependencies = [ "walkdir", ] +[[package]] +name = "sqlbuild-cli" +version = "0.155.0" +dependencies = [ + "blake3", + "ctrlc", + "getrandom", + "rustix", + "serde", + "serde_json", + "sqlbuild-cache", + "sqlbuild-core", + "tempfile", +] + [[package]] name = "sqlbuild-config" version = "0.155.0" dependencies = [ "saphyr-parser", + "sqlbuild-core", "tempfile", "toml_edit", "toml_parser", @@ -759,6 +842,7 @@ dependencies = [ name = "sqlbuild-model-config" version = "0.155.0" dependencies = [ + "regex", "sqlbuild-core", ] @@ -769,6 +853,7 @@ dependencies = [ "fensu-policy", "mimalloc", "pyo3", + "serde", "serde_json", "sqlbuild-analysis", "sqlbuild-attachments", @@ -777,16 +862,32 @@ dependencies = [ "sqlbuild-core", "sqlbuild-discovery", "sqlbuild-model-config", + "sqlbuild-refactor", "sqlbuild-render", "sqlbuild-rules", "sqlbuild-scopes", "sqlbuild-sqltext", ] +[[package]] +name = "sqlbuild-refactor" +version = "0.155.0" +dependencies = [ + "serde", + "serde_json", + "sqlbuild-analysis", + "sqlbuild-config", + "sqlbuild-core", + "sqlbuild-model-config", + "sqlbuild-sqltext", + "tempfile", +] + [[package]] name = "sqlbuild-render" version = "0.155.0" dependencies = [ + "serde_json", "sqlbuild-cache", "sqlbuild-core", "tempfile", @@ -817,7 +918,12 @@ dependencies = [ name = "sqlbuild-scopes" version = "0.155.0" dependencies = [ + "serde_json", + "sha2", + "sqlbuild-config", "sqlbuild-core", + "sqlbuild-discovery", + "sqlbuild-render", "sqlbuild-sqltext", ] diff --git a/Cargo.toml b/Cargo.toml index 75f7a092a9..195c021848 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,8 @@ members = [ "crates/sqlbuild-attachments", "crates/sqlbuild-analysis", "crates/sqlbuild-rules", + "crates/sqlbuild-refactor", + "crates/sqlbuild-cli", "crates/sqlbuild-python", ] resolver = "2" @@ -25,12 +27,14 @@ order = [ "sqlbuild-sqltext", "sqlbuild-config", "sqlbuild-discovery", + "sqlbuild-render", "sqlbuild-scopes", "sqlbuild-model-config", - "sqlbuild-render", "sqlbuild-attachments", "sqlbuild-analysis", "sqlbuild-rules", + "sqlbuild-refactor", + "sqlbuild-cli", "sqlbuild-python", ] python-boundary = "sqlbuild-python" @@ -45,6 +49,8 @@ rust-version = "1.88" [workspace.dependencies] fensu-policy = "=0.22.0" +# `sqb` names compute log captures with random ids, as `uuid.uuid4().hex`. +getrandom = "0.4" globset = "0.4" # Windows only: tells NTFS junctions, which Python walks as directories, from symbolic links. junction = { version = "2.1.0", default-features = false } @@ -63,8 +69,12 @@ polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot. "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } +# Windows only: `sqb` ignores Ctrl+C while the Python CLI it started handles it. +ctrlc = "3" quote = "1" rayon = "1" +# Unix only: `sqb` shares the compute log prune lock with Python's `filelock` through `flock`. +rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] } saphyr-parser = "0.1" regex = "1" serde = { version = "1", features = ["derive"] } @@ -78,6 +88,7 @@ sqlbuild-config = { path = "crates/sqlbuild-config" } sqlbuild-core = { path = "crates/sqlbuild-core" } sqlbuild-discovery = { path = "crates/sqlbuild-discovery" } sqlbuild-render = { path = "crates/sqlbuild-render" } +sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } @@ -99,5 +110,12 @@ unused_must_use = "deny" [workspace.lints.clippy] await_holding_lock = "deny" +[profile.dev] +debug = "line-tables-only" + +[profile.dev.package."*"] +opt-level = 1 +debug = false + [profile.release] strip = "symbols" diff --git a/Makefile b/Makefile index b8dea2f83b..061966ff71 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,10 @@ type: uv run ty check src tests +.PHONY: dev-native +dev-native: + uv run --no-sync maturin develop --profile dev --skip-install + rust-check: cargo fmt --all --check cargo clippy --workspace --all-targets --all-features -- -D warnings diff --git a/crates/README.md b/crates/README.md index 161417bc63..9c066e697b 100644 --- a/crates/README.md +++ b/crates/README.md @@ -1,7 +1,7 @@ # Native crates SQLBuild's native code is a Cargo workspace that builds one Python extension module, -`sqlbuild._native`. Only `sqlbuild-python` knows about Python; every other crate is plain Rust +`sqlbuild._native`, and the `sqb`/`sqlbuild` executables in the `sqlbuild-cli` wheel. Only `sqlbuild-python` knows about Python; every other crate is plain Rust that can be tested and benchmarked with `cargo test` alone. | Crate | Responsibility | @@ -11,12 +11,14 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-sqltext` | Lexical SQL text without polyglot: comment, quote and parenthesis scanning, the rules quote policy, model header tokenization and matching, static variable substitution, static reference extraction, and the general reference scan with table-function call arguments under each adapter's lexical rules. | | `sqlbuild-config` | Configuration file reading without Python: `tomllib`-compatible TOML, PyYAML `safe_load`-compatible YAML 1.1, and typed project and local config readers for the fields discovery needs. Errors carry their kind and position; YAML outside the supported subset is an `Unsupported` error. | | `sqlbuild-discovery` | Native project discovery and the only implementation of the model, SQL test, scenario, source and schema file collections and the declaration layout: the shared directory walk with Python's glob and sort semantics, file reading with Python's newline handling and read errors, and parsing of authored files with Python's exact discovery messages. Results are plain data the Python discovery facade materialises. | -| `sqlbuild-scopes` | Declaration scopes: the scope index with Python's record orders and diagnostics, declaration visibility, relationship grants, the scope lookup groups, and the dialect-aware scan of SQL tests and scenarios for their expected models and top-level CTEs, with Python's scanner errors. Anything it cannot reproduce exactly defers to Python. | -| `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. | -| `sqlbuild-render` | Native rendering: the macro call scanner (a byte-for-byte port of Python's), splicing of rendered calls with code-point spans, and the in-compile memo of recorded macro calls and their replayable events, which can carry results across compiles through the shared native store. Anything it cannot reproduce exactly defers to Python. | -| `sqlbuild-attachments` | Compile attachments for tests, audits, sources, functions, scenarios and seeds: attached generic audit argument merge, raw and quoted argument rendering, and severity and run scope resolution. Anything it cannot reproduce exactly defers to Python, which also raises every attachment error. | -| `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, type normalization, and SQL-test and scenario extraction, planning and rendering. Type normalization defers to Python for anything it cannot reproduce exactly. | +| `sqlbuild-scopes` | Declaration scopes, visibility, relationship grants, placement, report queries and the offline scope cache, including dialect-aware SQL-test and scenario scans. | +| `sqlbuild-model-config` | MODEL configuration, columns and audits, template expansion over caller values, and dynamic-column name patterns using Rust regex syntax. | +| `sqlbuild-render` | Macro call scanning, argument evaluation, expansion, splicing, memoisation and persistent call-store keys. Python executes user macro bodies at the extension boundary. | +| `sqlbuild-attachments` | Native attachment of tests, audits, sources, functions, scenarios and seeds, including argument merging, validation and audit policy resolution. Adapter value rendering remains an extension boundary. | +| `sqlbuild-analysis` | SQL analysis over polyglot: query analysis, binding, semantic validation and usage, column references, type normalization, FAST/RICH lineage, and SQL-test/scenario extraction and compiler-side assembly. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | +| `sqlbuild-refactor` | Native rename/move plans, contract-adoption edits, MODEL/SCHEMA/YAML text edits and atomic staging/commit with rollback. Shared scope placement delegates to the native scope implementation; user Python source inspection remains at the Python AST boundary. | +| `sqlbuild-cli` | The `sqb` and `sqlbuild` executables: `--version`, unchanged compiles replayed from the shared reuse store with Python's exact streams, exit code and compute log capture, and a handoff of every other invocation to the installed `sqlbuild-python` script. Built as its own wheel (`crates/sqlbuild-cli/pyproject.toml`). | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | ## Layering @@ -25,13 +27,14 @@ Dependencies point one way, from the top of this graph to the bottom: ```text sqlbuild-python - -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments -> sqlbuild-render - -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-discovery -> sqlbuild-config + -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments + -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-core +sqlbuild-cli -> sqlbuild-cache, sqlbuild-core ``` -`sqlbuild-config` and `sqlbuild-model-config` do not depend on the crates below them today, and -`sqlbuild-scopes` uses only `sqlbuild-sqltext` and `sqlbuild-core`; their place in the order fixes which crates may +`sqlbuild-scopes` reuses the macro scanner in `sqlbuild-render`; render itself depends only on +`sqlbuild-cache` and `sqlbuild-core`. Their place in the order fixes which crates may use them. The JSON emitter lives in `sqlbuild-core` so that any layer can produce text that must equal Python's `json.dumps` output. @@ -62,10 +65,11 @@ stages run natively: | `native-preview` | Opt-in: shipped stages plus stages still in development (`preview` tier). | The Python compiler and its `python` engine were removed once discovery, rendering and analysis -shipped natively; setting `python` is an error that says so. Native stages still hand inputs they -cannot reproduce exactly back to Python helpers (the allow-list in -`scripts/compiler_differential/native_fallbacks.toml` lists every such deferral the corpus -reaches). A failing render raises its first error directly and runs each macro call at most +shipped natively; setting `python` is an error that says so. Generic compiler stages do not +rescue native failures through a Python implementation. User macros, custom rules and adapter +callbacks retain their explicit Python boundaries. The recorder in +`scripts/compiler_differential/native_fallbacks.toml` tracks native answers and the remaining +extension/test-mock callbacks. A failing render raises its first error directly and runs each macro call at most once; a divergence the bridge detects is reported as a native stage mismatch. Because the macro-call store is always on, the [macro determinism contract](../website/src/content/docs/docs/concepts/macros.mdx) applies to diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index ff48a44396..d9f8c47650 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -20,8 +20,10 @@ regex.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true +sqlbuild-attachments.workspace = true sqlbuild-cache.workspace = true sqlbuild-core.workspace = true +sqlbuild-discovery.workspace = true sqlbuild-model-config.workspace = true sqlbuild-render.workspace = true sqlbuild-scopes.workspace = true diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs index 9417d4071e..38e4ee8cf1 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/constants.rs @@ -234,3 +234,17 @@ pub(crate) const CTE_ALIAS_PAYLOAD_KEY: &str = "alias"; pub(crate) const NAME_PAYLOAD_KEY: &str = "name"; pub(crate) const SCHEMA_TABLE_ARG: &str = "schema"; pub(crate) const CATALOG_TABLE_ARG: &str = "catalog"; +/// The set-operation keywords `\b(?:UNION|INTERSECT|EXCEPT)\b` searches for, in lowercase. +pub(crate) const SET_OPERATION_KEYWORDS: [&str; 3] = ["union", "intersect", "except"]; +/// The analysis relation name prefix of a table-function call. +pub(crate) const TABLE_FUNCTION_ANALYSIS_PREFIX: &str = "__sqlbuild_table_function_"; +pub(crate) const REF_KIND: &str = "ref"; +pub(crate) const UDF_KIND: &str = "udf"; +pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn"; +/// Python's `_LINEAGE_RESOURCE_TYPES`: reference kind and the resource type lineage names. +pub(crate) const LINEAGE_RESOURCE_TYPES: [(&str, &str); 4] = [ + ("ref", "model"), + ("source", "source"), + ("seed", "seed"), + ("table_fn", "table_fn"), +]; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs index a2aee74abf..f04bab938b 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/mod.rs @@ -3,6 +3,8 @@ pub mod expression_shapes; pub mod finish_analysis_session; pub mod finished_fact_models; pub mod finished_model_facts; +pub mod model_reference_facts; +pub mod names_set_operation; pub mod projection_facts; pub mod prove_dynamic_contracts; pub mod prove_finished_dynamic_contracts; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/model_reference_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/model_reference_facts.rs new file mode 100644 index 0000000000..30a3f51dcc --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/model_reference_facts.rs @@ -0,0 +1,48 @@ +//! A model's analysis references, lineage reference map and required relation names. + +use std::collections::BTreeSet; + +use crate::assembly::analysis_session::constants::{ + LINEAGE_RESOURCE_TYPES, REF_KIND, TABLE_FUNCTION_ANALYSIS_PREFIX, TABLE_FUNCTION_KIND, UDF_KIND, +}; +use crate::assembly::analysis_session::models::{ModelReference, ModelReferenceFacts}; + +/// The reference facts of `(kind, name)` rows, as Python's model analysis rows derive them. +pub fn model_reference_facts(rows: &[(String, String)]) -> ModelReferenceFacts { + let mut facts = ModelReferenceFacts::default(); + let mut required: BTreeSet = BTreeSet::new(); + for (kind, name) in rows { + let analysis_name = if kind == TABLE_FUNCTION_KIND { + format!("{TABLE_FUNCTION_ANALYSIS_PREFIX}{name}") + } else { + name.clone() + }; + facts.references.push(ModelReference { + analysis_name: analysis_name.clone(), + model_ref: kind == REF_KIND, + }); + if let Some((_, resource_type)) = LINEAGE_RESOURCE_TYPES + .iter() + .find(|(lineage_kind, _)| lineage_kind == kind) + { + let entry = ( + analysis_name.clone(), + (*resource_type).to_owned(), + name.clone(), + ); + match facts + .lineage_references + .iter_mut() + .find(|(existing, _, _)| *existing == analysis_name) + { + Some(existing) => *existing = entry, + None => facts.lineage_references.push(entry), + } + } + if kind != UDF_KIND { + required.insert(analysis_name); + } + } + facts.required_names = required.into_iter().collect(); + facts +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/main/names_set_operation.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/names_set_operation.rs new file mode 100644 index 0000000000..fc80c4da05 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/main/names_set_operation.rs @@ -0,0 +1,35 @@ +//! Python's `re.search(r"\b(?:UNION|INTERSECT|EXCEPT)\b", sql, re.IGNORECASE) is not None`. + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::is_python_word::is_python_word; +use sqlbuild_core::text::main::python_ignorecase_key::python_ignorecase_key; + +use crate::assembly::analysis_session::constants::SET_OPERATION_KEYWORDS; + +/// Whether `sql` names a set-operation keyword as a whole word, in any case. +pub fn names_set_operation(sql: &str) -> bool { + let python = active_python_text(); + let characters: Vec = sql.chars().collect(); + for start in 0..characters.len() { + if start > 0 && is_python_word(python, characters[start - 1]) { + continue; + } + for keyword in SET_OPERATION_KEYWORDS { + let end = start + keyword.len(); + if end > characters.len() { + continue; + } + let matches = characters[start..end] + .iter() + .zip(keyword.chars()) + .all(|(character, letter)| python_ignorecase_key(python, *character) == letter); + let bounded = characters + .get(end) + .is_none_or(|next| !is_python_word(python, *next)); + if matches && bounded { + return true; + } + } + } + false +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index bd0a141b75..a99190b5d9 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -14,7 +14,7 @@ use crate::assembly::analysis_session::types::{NullabilityRuleFn, OutputSources, use crate::semantic_validation::types::DiagnosticRow; /// One `ref`, `source`, `seed`, `table_fn` or `udf` call a model's SQL makes. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ModelReference { /// Python's `_analysis_reference_name`. pub analysis_name: String, @@ -44,6 +44,14 @@ pub struct ModelRequest { pub dynamic_families: Vec, } +/// The reference-derived fields of a `ModelRequest`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ModelReferenceFacts { + pub references: Vec, + pub lineage_references: Vec<(String, String, String)>, + pub required_names: Vec, +} + /// One declared family of columns a runtime-dynamic pivot generates. #[derive(Debug, Clone, PartialEq, Eq)] pub struct DynamicFamily { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs index f9737f30b6..0cc551662e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests.rs @@ -10,5 +10,7 @@ mod helpers; mod mappings; #[path = "tests/test_model_columns.rs"] mod model_columns; +#[path = "tests/test_reference_facts.rs"] +mod reference_facts; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_reference_facts.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_reference_facts.rs new file mode 100644 index 0000000000..9d698ba62b --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_reference_facts.rs @@ -0,0 +1,56 @@ +use crate::assembly::analysis_session::main::model_reference_facts::model_reference_facts; +use crate::assembly::analysis_session::tests::test_types::ReferenceFactsTestCase; + +#[test] +fn given_reference_rows_when_deriving_facts_then_python_lineage_map_and_required_names_match() { + let test_cases = [ + ReferenceFactsTestCase { + description: "a repeated name keeps its first position with the last resource type", + rows: &[("ref", "orders"), ("seed", "codes"), ("source", "orders")], + expected_lineage: &[("orders", "source", "orders"), ("codes", "seed", "codes")], + expected_required: &["codes", "orders"], + }, + ReferenceFactsTestCase { + description: "table functions are renamed; UDFs and dbt refs are not lineage", + rows: &[ + ("table_fn", "expand"), + ("udf", "scale"), + ("dbt_ref", "legacy"), + ], + expected_lineage: &[("__sqlbuild_table_function_expand", "table_fn", "expand")], + expected_required: &["__sqlbuild_table_function_expand", "legacy"], + }, + ]; + + for test_case in test_cases { + let rows: Vec<(String, String)> = test_case + .rows + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect(); + + let facts = model_reference_facts(&rows); + + let lineage: Vec<(&str, &str, &str)> = facts + .lineage_references + .iter() + .map(|(a, b, c)| (a.as_str(), b.as_str(), c.as_str())) + .collect(); + assert_eq!( + ( + lineage, + facts + .required_names + .iter() + .map(String::as_str) + .collect::>() + ), + ( + test_case.expected_lineage.to_vec(), + test_case.expected_required.to_vec() + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index cf81b59c4f..8a81d47677 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -170,3 +170,10 @@ pub(crate) struct ModelColumnsTestCase { pub(crate) poisoned: &'static [&'static str], pub(crate) expected_columns: ExpectedModelColumns, } + +pub(crate) struct ReferenceFactsTestCase { + pub(crate) description: &'static str, + pub(crate) rows: &'static [(&'static str, &'static str)], + pub(crate) expected_lineage: &'static [(&'static str, &'static str, &'static str)], + pub(crate) expected_required: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs index de5150607b..f49044ae83 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/deps.rs @@ -2,6 +2,7 @@ use std::collections::HashSet; +use sqlbuild_core::text::main::active_python_text::active_python_text; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::assembly::project::constants::{ @@ -22,14 +23,10 @@ pub(crate) fn audit_deps(audit: &AuditFacts) -> Fact> { let mut keys: Vec = reference_deps(&audit.references); if let Some((kind, name)) = &audit.attached { if !ATTACHED_AUDIT_KINDS.contains(&kind.as_str()) { - return Err(match python_str_repr(kind) { - Some(repr) => { - AssemblyError::Value(format!("{repr} is not a valid {AUDIT_TARGET_KIND_ENUM}")) - } - None => AssemblyError::Internal(format!( - "a non-ASCII attached audit target kind is not a valid {AUDIT_TARGET_KIND_ENUM}" - )), - }); + let repr: String = python_str_repr(active_python_text(), kind); + return Err(AssemblyError::Value(format!( + "{repr} is not a valid {AUDIT_TARGET_KIND_ENUM}" + ))); } keys.push((kind.clone(), name.clone())); } diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs index ff59b71794..c2a8ebb91a 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/targets.rs @@ -114,7 +114,7 @@ pub(crate) fn seed_target( } /// `validate_preserved_logical_namespace` for the resource Python names `resource_label`. -fn preserved_namespace( +pub(crate) fn preserved_namespace( resource_label: &str, target: Option<&TargetNamespace>, logical_database: Option<&String>, diff --git a/crates/sqlbuild-analysis/src/assembly/project/constants.rs b/crates/sqlbuild-analysis/src/assembly/project/constants.rs index 0eedde351b..cb4c75ce95 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/constants.rs @@ -43,3 +43,7 @@ pub(crate) const UNLISTED_ENVIRONMENT: &str = "a target template read an environment variable the request did not list"; /// `AttachedAuditTargetKind`'s name in the `ValueError` its constructor raises. pub(crate) const AUDIT_TARGET_KIND_ENUM: &str = "AttachedAuditTargetKind"; +/// The prefix of an environment read in a target template. +pub(crate) const ENVIRONMENT_READ_PREFIX: &str = "ENV:"; +/// Characters that end an `ENV:` name besides Python whitespace. +pub(crate) const ENVIRONMENT_NAME_TERMINATORS: [char; 6] = ['(', ')', ',', '\'', '"', '}']; diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/environment_template_names.rs b/crates/sqlbuild-analysis/src/assembly/project/main/environment_template_names.rs new file mode 100644 index 0000000000..5446757144 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/environment_template_names.rs @@ -0,0 +1,34 @@ +//! The `ENV:` names target templates read, as `re.findall(r"ENV:([^\s(),'\"}]+)")` finds them. + +use sqlbuild_core::text::main::is_python_space::is_python_space; + +use crate::assembly::project::constants::{ENVIRONMENT_NAME_TERMINATORS, ENVIRONMENT_READ_PREFIX}; + +/// Every distinct `ENV:` name across `values`, in first-seen order. +pub fn environment_template_names(values: &[Option]) -> Vec { + let mut names: Vec = Vec::new(); + for value in values.iter().flatten() { + let mut position = 0; + while let Some(found) = value[position..].find(ENVIRONMENT_READ_PREFIX) { + let start = position + found + ENVIRONMENT_READ_PREFIX.len(); + let length: usize = value[start..] + .chars() + .take_while(|character| { + !is_python_space(*character) + && !ENVIRONMENT_NAME_TERMINATORS.contains(character) + }) + .map(char::len_utf8) + .sum(); + if length == 0 { + position += found + 1; + continue; + } + let name = &value[start..start + length]; + if !names.iter().any(|existing| existing == name) { + names.push(name.to_owned()); + } + position = start + length; + } + } + names +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs index dccf961e02..9cbfd9734e 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/main/mod.rs @@ -1,3 +1,5 @@ pub mod assemble_project_resources; pub mod check_sql_syntax; +pub mod environment_template_names; +pub mod preserved_logical_namespace; pub mod sql_syntax_error; diff --git a/crates/sqlbuild-analysis/src/assembly/project/main/preserved_logical_namespace.rs b/crates/sqlbuild-analysis/src/assembly/project/main/preserved_logical_namespace.rs new file mode 100644 index 0000000000..ed80380e62 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/main/preserved_logical_namespace.rs @@ -0,0 +1,14 @@ +//! `validate_preserved_logical_namespace`: a target that preserves needs a logical namespace. + +use crate::assembly::project::_helpers::targets::preserved_namespace; +use crate::assembly::project::errors::AssemblyError; +use crate::assembly::project::models::TargetNamespace; + +/// The error Python raises when `target` preserves a dimension the resource has no logical value for. +pub fn preserved_logical_namespace( + resource_label: &str, + target: Option<&TargetNamespace>, + logical: (Option<&String>, Option<&String>), +) -> Result<(), AssemblyError> { + preserved_namespace(resource_label, target, logical.0, logical.1) +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests.rs b/crates/sqlbuild-analysis/src/assembly/project/tests.rs index b8a5f96c5e..48e3358156 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_environment_names.rs"] +mod environment_names; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_project_assembly.rs"] diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_environment_names.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_environment_names.rs new file mode 100644 index 0000000000..c1c740e6d9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_environment_names.rs @@ -0,0 +1,39 @@ +use crate::assembly::project::main::environment_template_names::environment_template_names; +use crate::assembly::project::tests::test_types::EnvironmentNamesTestCase; + +#[test] +fn given_template_values_when_scanning_then_environment_names_match_python_findall() { + let test_cases = [ + EnvironmentNamesTestCase { + description: "names end at whitespace and template punctuation, deduplicated", + values: &[ + Some("${ENV:WAREHOUSE}_db"), + None, + Some("ENV:a ENV:b,ENV:WAREHOUSE"), + ], + expected_names: &["WAREHOUSE", "a", "b"], + }, + EnvironmentNamesTestCase { + description: "an empty name is skipped and a nested prefix is part of the name", + values: &[Some("ENV:)ENV:ENV:x\u{3000}y")], + expected_names: &["ENV:x"], + }, + ]; + + for test_case in test_cases { + let values: Vec> = test_case + .values + .iter() + .map(|value| value.map(str::to_owned)) + .collect(); + + let names = environment_template_names(&values); + + assert_eq!( + names, + test_case.expected_names.to_vec(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs index adfbe725ba..88b3291f57 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/tests/test_types.rs @@ -80,3 +80,9 @@ pub(crate) struct SyntaxBatchTestCase { /// None where Python raises. pub(crate) expected_valid: Option, } + +pub(crate) struct EnvironmentNamesTestCase { + pub(crate) description: &'static str, + pub(crate) values: &'static [Option<&'static str>], + pub(crate) expected_names: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/binding_locations/constants.rs b/crates/sqlbuild-analysis/src/binding_locations/constants.rs new file mode 100644 index 0000000000..9bc5d6c902 --- /dev/null +++ b/crates/sqlbuild-analysis/src/binding_locations/constants.rs @@ -0,0 +1,6 @@ +//! Binding codes and severities the location pass reads. + +/// The internal unknown-table code, which compile never reports. +pub(crate) const UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; +/// The severity whose repeats at a known span compile drops. +pub(crate) const ERROR_SEVERITY: &str = "error"; diff --git a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs new file mode 100644 index 0000000000..80221f225b --- /dev/null +++ b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs @@ -0,0 +1,90 @@ +//! A model's binding diagnostics at their authored locations, deduplicated as compile reports. + +use std::collections::HashSet; + +use crate::binding_locations::constants::{ERROR_SEVERITY, UNKNOWN_TABLE_INTERNAL_CODE}; +use crate::binding_locations::models::{AuthoredSpan, BindingSource, LocatedDiagnostic}; +use crate::semantic_validation::models::{BindingPositions, PositionInput}; +use crate::semantic_validation::types::DiagnosticRow; + +/// Each reported diagnostic in order; an error mapping a position is the mapper's message. +pub fn locate_binding_diagnostics( + rows: &[DiagnosticRow], + source: &BindingSource<'_>, +) -> Result, String> { + let line_offset: Option = source + .authored + .find(source.query) + .map(|start| source.authored[..start].matches('\n').count()); + let mut positions: Option = None; + let mut seen: HashSet<(String, String, Option, Option)> = HashSet::new(); + let mut located: Vec = Vec::new(); + for (code, message, line, column, start, end, severity) in rows { + if code == UNKNOWN_TABLE_INTERNAL_CODE { + continue; + } + let mapper: &BindingPositions = match &mut positions { + Some(mapper) => mapper, + slot @ None => slot.insert(BindingPositions::new(position_input(source))?), + }; + let location: Option = + authored_span(mapper, (*start, *line, *column), *end, message)?; + let (line, column) = match location { + Some(span) => (Some(span.line), Some(span.column)), + None => ( + match (line, line_offset) { + (Some(line), Some(offset)) => Some(line + offset), + (line, _) => *line, + }, + *column, + ), + }; + let key = (code.clone(), message.clone(), line, column); + if seen.contains(&key) && start.is_some() && severity == ERROR_SEVERITY { + continue; + } + seen.insert(key); + located.push(LocatedDiagnostic { + code: code.clone(), + message: message.clone(), + severity: severity.clone(), + line, + column, + location, + }); + } + Ok(located) +} + +fn position_input(source: &BindingSource<'_>) -> PositionInput { + let (expanded, passes) = source.expansion.unwrap_or((source.query, &[])); + PositionInput { + authored: source.authored.to_owned(), + query: source.query.to_owned(), + expanded: expanded.to_owned(), + cleaned: source.cleaned.to_owned(), + passes: passes.to_vec(), + } +} + +/// `get_authored_binding_location`: the start, and the end when the diagnostic spans one. +fn authored_span( + mapper: &BindingPositions, + (start, line, column): (Option, Option, Option), + end: Option, + message: &str, +) -> Result, String> { + let (Some(line), Some(column)) = mapper.position(start, line, column, message)? else { + return Ok(None); + }; + let (end_line, end_column) = match end { + Some(end) => mapper.position(Some(end), None, None, "")?, + None => (None, None), + }; + Ok(Some(AuthoredSpan { + line, + column, + end_line, + end_column, + })) +} diff --git a/crates/sqlbuild-analysis/src/binding_locations/main/mod.rs b/crates/sqlbuild-analysis/src/binding_locations/main/mod.rs new file mode 100644 index 0000000000..261c9b6f29 --- /dev/null +++ b/crates/sqlbuild-analysis/src/binding_locations/main/mod.rs @@ -0,0 +1 @@ +pub mod locate_binding_diagnostics; diff --git a/crates/sqlbuild-analysis/src/binding_locations/mod.rs b/crates/sqlbuild-analysis/src/binding_locations/mod.rs new file mode 100644 index 0000000000..1adbc61cd0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/binding_locations/mod.rs @@ -0,0 +1,8 @@ +//! Authored locations of a model's binding diagnostics, as compile reports them. + +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/binding_locations/models.rs b/crates/sqlbuild-analysis/src/binding_locations/models.rs new file mode 100644 index 0000000000..292ed5690a --- /dev/null +++ b/crates/sqlbuild-analysis/src/binding_locations/models.rs @@ -0,0 +1,36 @@ +//! A model's authored SQL texts and the located diagnostics read from them. + +use crate::semantic_validation::types::Expansion; + +/// The texts one model's binding diagnostics are mapped back through. +#[derive(Clone, Copy, Debug)] +pub struct BindingSource<'a> { + /// The whole authored model file. + pub authored: &'a str, + /// The authored query below the header. + pub query: &'a str, + /// The SQL the binding pass validated. + pub cleaned: &'a str, + /// The authored expansion's expanded SQL and passes, when interpolation left the query as is. + pub expansion: Option<(&'a str, &'a [Vec])>, +} + +/// A diagnostic's authored span: start line and column, and its end when it has one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AuthoredSpan { + pub line: usize, + pub column: usize, + pub end_line: Option, + pub end_column: Option, +} + +/// One binding diagnostic as compile reports it for a model. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LocatedDiagnostic { + pub code: String, + pub message: String, + pub severity: String, + pub line: Option, + pub column: Option, + pub location: Option, +} diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests.rs b/crates/sqlbuild-analysis/src/binding_locations/tests.rs new file mode 100644 index 0000000000..54f95783b9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/binding_locations/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_locate_binding_diagnostics.rs"] +mod locate_binding_diagnostics; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs b/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs new file mode 100644 index 0000000000..a09c0b76aa --- /dev/null +++ b/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs @@ -0,0 +1,58 @@ +use crate::semantic_validation::types::DiagnosticRow; + +pub(super) const MODEL: &str = + "MODEL (\n description \"Orders\"\n);\n\nSELECT o.amount, o.missing\nFROM orders AS o"; +pub(super) const QUERY: &str = "SELECT o.amount, o.missing\nFROM orders AS o"; + +fn row( + code: &str, + message: &str, + position: (Option, Option), + span: (Option, Option), + severity: &str, +) -> DiagnosticRow { + ( + code.to_owned(), + message.to_owned(), + position.0, + position.1, + span.0, + span.1, + severity.to_owned(), + ) +} + +fn missing_column() -> DiagnosticRow { + row( + "E201", + "Unknown column 'missing'", + (Some(1), Some(17)), + (Some(17), Some(24)), + "error", + ) +} + +pub(super) fn one_missing_column() -> Vec { + vec![missing_column()] +} + +pub(super) fn repeated_rows() -> Vec { + let warning: DiagnosticRow = row("W1", "warn", (Some(1), Some(1)), (None, None), "warning"); + vec![ + row("B000", "x", (Some(1), Some(1)), (Some(0), Some(1)), "error"), + missing_column(), + missing_column(), + warning.clone(), + warning, + ] +} + +pub(super) fn unmapped_row() -> Vec { + vec![row( + "E1", + "no position", + (Some(2), Some(3)), + (None, None), + "error", + )] +} diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests/test_locate_binding_diagnostics.rs b/crates/sqlbuild-analysis/src/binding_locations/tests/test_locate_binding_diagnostics.rs new file mode 100644 index 0000000000..1fbed867ce --- /dev/null +++ b/crates/sqlbuild-analysis/src/binding_locations/tests/test_locate_binding_diagnostics.rs @@ -0,0 +1,70 @@ +use crate::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; +use crate::binding_locations::models::BindingSource; +use crate::binding_locations::tests::helpers::{ + MODEL, QUERY, one_missing_column, repeated_rows, unmapped_row, +}; +use crate::binding_locations::tests::test_types::LocateBindingDiagnosticsTestCase; + +#[test] +fn given_binding_rows_when_locating_then_python_locations_and_deduplication_hold() { + let test_cases = [ + LocateBindingDiagnosticsTestCase { + description: "a column error maps to its authored span below the header", + authored: MODEL, + query: QUERY, + rows: one_missing_column, + expected_located: &[("E201", Some(5), Some(20), Some((5, 20, Some(5), Some(25))))], + }, + LocateBindingDiagnosticsTestCase { + description: "internal codes drop, spanned error repeats drop, other repeats stay", + authored: MODEL, + query: QUERY, + rows: repeated_rows, + expected_located: &[ + ("E201", Some(5), Some(20), Some((5, 20, Some(5), Some(25)))), + ("W1", Some(5), Some(1), Some((5, 1, None, None))), + ("W1", Some(5), Some(1), Some((5, 1, None, None))), + ], + }, + LocateBindingDiagnosticsTestCase { + description: "a query missing from the file keeps the raw line and column", + authored: "SELECT 1", + query: "absent query", + rows: unmapped_row, + expected_located: &[("E1", Some(2), Some(3), None)], + }, + ]; + for test_case in test_cases { + let source: BindingSource<'_> = BindingSource { + authored: test_case.authored, + query: test_case.query, + cleaned: test_case.query, + expansion: None, + }; + + let located = locate_binding_diagnostics(&(test_case.rows)(), &source) + .expect("positions map") + .into_iter() + .map(|item| { + ( + item.code, + item.line, + item.column, + item.location + .map(|span| (span.line, span.column, span.end_line, span.end_column)), + ) + }) + .collect::>(); + + assert_eq!( + located, + test_case + .expected_located + .iter() + .map(|(code, line, column, span)| ((*code).to_owned(), *line, *column, *span)) + .collect::>(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs b/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs new file mode 100644 index 0000000000..300b0e43f8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs @@ -0,0 +1,18 @@ +use crate::semantic_validation::types::DiagnosticRow; + +/// `(code, line, column, (line, column, end line, end column))` of one reported diagnostic. +pub(super) type LocatedCase = ( + &'static str, + Option, + Option, + Option<(usize, usize, Option, Option)>, +); + +/// A model's texts, its raw binding rows, and the diagnostics compile reports for them. +pub(super) struct LocateBindingDiagnosticsTestCase { + pub(super) description: &'static str, + pub(super) authored: &'static str, + pub(super) query: &'static str, + pub(super) rows: fn() -> Vec, + pub(super) expected_located: &'static [LocatedCase], +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/declaration_errors.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/declaration_errors.rs new file mode 100644 index 0000000000..a8623c6bd0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/declaration_errors.rs @@ -0,0 +1,85 @@ +//! The messages enum and constant expansion raises, worded as the attachment always has. + +use crate::compiled_project::constants::{ + DECLARATION_EXPANSION_CONTEXT, INVALID_CONSTANT_REFERENCE_STOP, INVALID_ENUM_REFERENCE_STOP, + UNCLOSED_BLOCK_COMMENT_STOP, +}; +use crate::compiled_project::models::InaccessibleDeclaration; + +pub(crate) fn unknown_declaration( + kind: &str, + name: &str, + consumer: &str, + visible: &[String], +) -> String { + let scope_help: &str = if name.starts_with('_') { + " in this model" + } else { + "" + }; + let mut names: Vec<&str> = visible.iter().map(String::as_str).collect(); + names.sort_unstable(); + let listed: String = if names.is_empty() { + "none".to_owned() + } else { + names.join(", ") + }; + format!("Unknown {kind} '{name}'{scope_help} in '{consumer}'. Visible {kind}s: {listed}") +} + +pub(crate) fn unknown_member( + name: &str, + member: &str, + consumer: &str, + members: &[String], +) -> String { + format!( + "Unknown member '{member}' for enum '{name}' in '{consumer}'. Available members: {}", + members.join(", ") + ) +} + +pub(crate) fn inaccessible( + kind: &str, + name: &str, + consumer: &str, + record: &InaccessibleDeclaration, +) -> String { + let owner_identity: String = record + .owner_identity + .as_ref() + .map(|(owner_kind, owner_name)| format!(" ({owner_kind} '{owner_name}')")) + .unwrap_or_default(); + format!( + "{} '{name}' is known but inaccessible in '{consumer}'. It is defined at {}:{}:{} with \ + {} scope owned by '{}'{owner_identity}; consumer path: '{consumer}'", + capitalized(kind), + record.path, + record.line, + record.column, + record.scope, + record.owner, + ) +} + +pub(crate) fn scan_stop(code: u8, consumer: &str) -> String { + match code { + INVALID_ENUM_REFERENCE_STOP => format!("Invalid enum reference in '{consumer}'"), + INVALID_CONSTANT_REFERENCE_STOP => format!("Invalid constant reference in '{consumer}'"), + UNCLOSED_BLOCK_COMMENT_STOP => { + format!("{DECLARATION_EXPANSION_CONTEXT} contains an unclosed block comment") + } + _ => format!("{DECLARATION_EXPANSION_CONTEXT} contains an unclosed quoted string"), + } +} + +/// `str.capitalize`: the first character upper-cased, the rest lower-cased. +fn capitalized(text: &str) -> String { + let mut characters = text.chars(); + characters.next().map_or_else(String::new, |first| { + first + .to_uppercase() + .chain(characters.flat_map(char::to_lowercase)) + .collect() + }) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/locating.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/locating.rs new file mode 100644 index 0000000000..27d74ef763 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/locating.rs @@ -0,0 +1,146 @@ +//! Locating a reference-scan fault of expanded model SQL in the authored model file. + +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_sqltext::compiler::main::model_header_matching::match_batch; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, ReferenceScanFailure}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::compiled_project::_helpers::sources::char_position; +use crate::compiled_project::models::{ScanFaultOrigin, SplicedSpan}; + +/// `_located_scan_error`'s message: `path:line:column: message`, or `path: message`. +pub(crate) fn located_scan_error( + failure: &ReferenceScanFailure, + sql: &str, + origin: &ScanFaultOrigin<'_>, + syntax: &LexicalSyntax, +) -> String { + let path: &str = origin.relative_path; + match authored_fault_offset(failure, sql, origin, syntax) { + Some(offset) => { + let contents: Vec = origin.contents.chars().collect(); + let position = char_position(&contents, offset); + format!( + "{path}:{}:{}: {}", + position.line, position.column, failure.message + ) + } + None => format!("{path}: {}", failure.message), + } +} + +fn authored_fault_offset( + failure: &ReferenceScanFailure, + sql: &str, + origin: &ScanFaultOrigin<'_>, + syntax: &LexicalSyntax, +) -> Option { + let contents: Vec = origin.contents.chars().collect(); + let passes: &[Vec] = origin.passes.as_deref()?; + let sql_characters: Vec = sql.chars().collect(); + let start: usize = failure.start; + let body_start: usize = model_query_start(origin.contents, origin.query_sql)?; + if start >= sql_characters.len() { + return None; + } + let (mapped, generated) = map_through_passes(start, passes); + let authored: usize = body_start + mapped; + if generated || authored >= contents.len() || contents[authored] != sql_characters[start] { + return None; + } + let mut blanked: Vec = sql_characters; + for (output_start, output_end) in expansion_output_ranges(passes) { + let length: usize = blanked.len(); + let from: usize = output_start.min(length); + let to: usize = output_end.min(length).max(from); + let spaces = std::iter::repeat_n(' ', output_end.saturating_sub(output_start)); + blanked.splice(from..to, spaces); + } + let blanked: String = blanked.into_iter().collect(); + match extract_sql_references(&blanked, syntax) { + ReferenceExtraction::Failed(rescanned) if rescanned == *failure => Some(authored), + _ => None, + } +} + +/// `get_model_query_start`: where the stripped query starts after the MODEL header, if it does. +pub(crate) fn model_query_start(contents: &str, query_sql: &str) -> Option { + let (_, _, sql_start) = match_batch(&[contents.to_owned()]).pop().flatten()?; + let characters: Vec = contents.chars().collect(); + let mut start: usize = sql_start; + while start < characters.len() && is_python_space(characters[start]) { + start += 1; + } + let query: Vec = query_sql.chars().collect(); + characters[start..].starts_with(&query).then_some(start) +} + +/// `map_through_passes`: the offset in the first pass's source, and whether expansion made it. +pub(crate) fn map_through_passes(offset: usize, passes: &[Vec]) -> (usize, bool) { + let mut current: usize = offset; + let mut generated: bool = false; + for spans in passes.iter().rev() { + let (mapped, made) = map_output_offset(current, spans); + current = mapped; + generated |= made; + } + (current, generated) +} + +fn map_output_offset(offset: usize, spans: &[SplicedSpan]) -> (usize, bool) { + let mut mapped: isize = to_signed(offset); + for span in spans { + if offset < span.output_start { + break; + } + if offset < span.output_end { + return (span.source_start, true); + } + mapped += to_signed(span.source_end) + - to_signed(span.source_start) + - (to_signed(span.output_end) - to_signed(span.output_start)); + } + (usize::try_from(mapped).unwrap_or_default(), false) +} + +/// `_expansion_output_ranges`: every expansion's output range in the final pass's output. +pub(crate) fn expansion_output_ranges(passes: &[Vec]) -> Vec<(usize, usize)> { + let mut ranges: Vec<(usize, usize)> = Vec::new(); + for (index, spans) in passes.iter().enumerate() { + for span in spans { + let mut output_start: usize = span.output_start; + let mut output_end: usize = span.output_end; + for later in &passes[index + 1..] { + output_start = later_pass_offset(output_start, later, false); + output_end = later_pass_offset(output_end, later, true); + } + ranges.push((output_start, output_end)); + } + } + ranges +} + +fn later_pass_offset(offset: usize, spans: &[SplicedSpan], end: bool) -> usize { + let mut shift: isize = 0; + for span in spans { + if offset < span.source_start || (end && offset == span.source_start) { + break; + } + if offset < span.source_end || (end && offset == span.source_end) { + return if end { + span.output_end + } else { + span.output_start + }; + } + shift += to_signed(span.output_end) + - to_signed(span.output_start) + - (to_signed(span.source_end) - to_signed(span.source_start)); + } + usize::try_from(to_signed(offset) + shift).unwrap_or_default() +} + +fn to_signed(value: usize) -> isize { + isize::try_from(value).unwrap_or(isize::MAX) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/mod.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/mod.rs new file mode 100644 index 0000000000..d0e6655b43 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod declaration_errors; +pub(crate) mod locating; +pub(crate) mod plain; +pub(crate) mod sources; +pub(crate) mod splice; +pub(crate) mod store; diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs new file mode 100644 index 0000000000..e0f79d49e6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs @@ -0,0 +1,43 @@ +//! The model loop's ordered decisions and the facts it retains for a finished model. + +use crate::compiled_project::constants::MACRO_TOKEN; +use crate::compiled_project::models::{ + AnalysisGate, AnalysisOverride, CompiledModelFacts, ModelLoopSettings, PlainModel, +}; + +/// The attachment's gate: project switches first, so an override is only read when they allow. +pub(crate) fn analysis_gate( + settings: ModelLoopSettings, + analysis_override: AnalysisOverride, +) -> AnalysisGate { + if !settings.sql_analysis || settings.no_sql_validation { + return AnalysisGate::Disabled; + } + match analysis_override { + AnalysisOverride::Inherit | AnalysisOverride::Enabled => AnalysisGate::Enabled, + AnalysisOverride::Disabled => AnalysisGate::Disabled, + AnalysisOverride::Conflict => AnalysisGate::Conflict, + } +} + +/// Whether declaration-expanded SQL still calls macros. +pub(crate) fn calls_macros(sql: &str) -> bool { + sql.contains(MACRO_TOKEN) +} + +pub(crate) fn plain_model_facts(model: PlainModel) -> CompiledModelFacts { + CompiledModelFacts { + schema: model.schema, + name: model.name, + relative_path: model.relative_path, + query_sql: model.query_sql, + authored_query_sql: model.authored_query_sql, + authored_sql: model.authored_sql, + config: model.config, + references: model.references, + enum_columns: model.enum_columns, + enum_declarations: model.enum_declarations, + constant_declarations: model.constant_declarations, + analysis: None, + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/sources.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/sources.rs new file mode 100644 index 0000000000..88f3e305a2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/sources.rs @@ -0,0 +1,99 @@ +//! Where a model's authored settings live: opt-out keys and legacy schema declarations. + +use sqlbuild_core::text::main::is_python_word::is_python_word; +use sqlbuild_core::text::models::PythonText; + +use crate::compiled_project::constants::{ + CURRENT_DIRECTORY, NEWLINE, OPT_OUT_KEYS, PATH_SEPARATOR, +}; +use crate::compiled_project::models::{LegacySchemaFile, TextPosition}; + +/// `_text_location`: the first whole-word opt-out key at or after character `start`. +pub(crate) fn opt_out_position(python: PythonText, text: &str, start: usize) -> TextPosition { + let characters: Vec = text.chars().collect(); + let mut offset: usize = 0; + for index in start..characters.len() { + if starts_key(python, &characters, index) { + offset = index; + break; + } + } + char_position(&characters, offset) +} + +/// The one-based line and column of character `offset`. +pub(crate) fn char_position(characters: &[char], offset: usize) -> TextPosition { + let offset: usize = offset.min(characters.len()); + let mut line: usize = 1; + let mut line_start: usize = 0; + for (index, character) in characters[..offset].iter().enumerate() { + if *character == NEWLINE { + line += 1; + line_start = index + 1; + } + } + TextPosition { + line, + column: offset - line_start + 1, + } +} + +/// Whether an opt-out key starts at `index` with a word boundary on both sides. +fn starts_key(python: PythonText, characters: &[char], index: usize) -> bool { + if index > 0 && is_python_word(python, characters[index - 1]) { + return false; + } + for key in OPT_OUT_KEYS { + let end: usize = index + key.chars().count(); + if end > characters.len() || !characters[index..end].iter().copied().eq(key.chars()) { + continue; + } + if characters + .get(end) + .is_none_or(|&next| !is_python_word(python, next)) + { + return true; + } + } + false +} + +/// The error for a model a legacy schema file under its directory still declares. +pub(crate) fn legacy_schema_error( + model: (&str, &str), + schema_files: &[LegacySchemaFile], +) -> Option { + let (name, relative_path) = model; + let model_parts: Vec<&str> = parts(relative_path); + let mut matches: Vec<&str> = Vec::new(); + for file in schema_files { + let directory: Vec<&str> = parts(&file.relative_path); + if !model_parts.starts_with(&directory[..directory.len().saturating_sub(1)]) { + continue; + } + for entry in &file.model_names { + if entry == name { + matches.push(&file.relative_path); + } + } + } + match matches.as_slice() { + [] => None, + [file] => Some(format!( + "Schema file {file} declares model '{name}', but model metadata must live in \ + {relative_path} MODEL(...). Move description, columns, and audits into the model \ + header." + )), + files => Some(format!( + "Model file {relative_path} matched multiple schema.yml declarations: {}", + files.join(", ") + )), + } +} + +/// A relative path's parts, as `PurePosixPath.parts` gives them (`.` has none). +fn parts(path: &str) -> Vec<&str> { + path.split(PATH_SEPARATOR) + .filter(|part| !part.is_empty() && *part != CURRENT_DIRECTORY) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/splice.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/splice.rs new file mode 100644 index 0000000000..e5a79a1a33 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/splice.rs @@ -0,0 +1,51 @@ +//! Replacing scanned declaration references, indexed by character as Python strings are. + +use crate::compiled_project::models::{DeclarationSplice, SplicedSpan, SplicedSql}; + +pub(crate) fn splice_declarations(sql: &str, splices: &[DeclarationSplice]) -> SplicedSql { + let characters: Vec = sql.chars().collect(); + let mut output: String = String::with_capacity(sql.len()); + let mut spans: Vec = Vec::with_capacity(splices.len()); + let mut cursor: usize = 0; + let mut output_length: usize = 0; + for splice in splices { + output.extend(&characters[cursor..splice.start]); + output_length += splice.start - cursor; + let replacement_length: usize = splice.replacement.chars().count(); + output.push_str(&splice.replacement); + spans.push(SplicedSpan { + source_start: splice.start, + source_end: splice.end, + output_start: output_length, + output_end: output_length + replacement_length, + }); + output_length += replacement_length; + cursor = splice.end; + } + output.extend(&characters[cursor..]); + SplicedSql { sql: output, spans } +} + +pub(crate) fn merge_call_site_references( + references: Vec, + arguments: Vec, +) -> Vec { + if arguments.is_empty() { + return references; + } + let mut merged: Vec = Vec::with_capacity(references.len() + arguments.len()); + for reference in references.into_iter().chain(arguments) { + if !merged.contains(&reference) { + merged.push(reference); + } + } + merged +} + +/// An enum member's SQL literal: integers as written, text single-quoted with quotes doubled. +pub(crate) fn enum_member_literal(text: &str, integer: bool) -> String { + if integer { + return text.to_owned(); + } + format!("'{}'", text.replace('\'', "''")) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/store.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/store.rs new file mode 100644 index 0000000000..37a08d71d0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/store.rs @@ -0,0 +1,36 @@ +//! Recording and reading model facts by name, keeping production order. + +use crate::compiled_project::models::{ + CompiledModelFacts, CompiledProjectFacts, ModelAnalysisFact, +}; + +impl CompiledProjectFacts { + /// Keep `model`; a model recorded again under the same name replaces it in place. + pub(crate) fn record_model(&mut self, model: CompiledModelFacts) { + match self.model_index.get(&model.name) { + Some(&position) => self.models[position] = model, + None => { + self.model_index + .insert(model.name.clone(), self.models.len()); + self.models.push(model); + } + } + } + + /// Attach a model's typed analysis; `false` when no model of that name was recorded. + pub(crate) fn record_analysis(&mut self, name: &str, analysis: ModelAnalysisFact) -> bool { + match self.model_index.get(name) { + Some(&position) => { + self.models[position].analysis = Some(analysis); + true + } + None => false, + } + } + + pub(crate) fn model(&self, name: &str) -> Option<&CompiledModelFacts> { + self.model_index + .get(name) + .map(|&position| &self.models[position]) + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/constants.rs b/crates/sqlbuild-analysis/src/compiled_project/constants.rs new file mode 100644 index 0000000000..83408513be --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/constants.rs @@ -0,0 +1,24 @@ +//! Config keys and SQL tokens the native model loop reads, as the Python attachment names them. + +/// Header keys attached as schema metadata and stripped from the model's config. +pub(crate) const HEADER_METADATA_KEYS: [&str; 8] = [ + "description", + "columns", + "dynamic_columns", + "model_schema", + "audits", + "audit_factories", + "enums", + "constants", +]; +pub(crate) const DECLARATION_EXPANSION_CONTEXT: &str = "Enum and constant expansion"; +pub(crate) const UNCLOSED_BLOCK_COMMENT_STOP: u8 = 1; +pub(crate) const INVALID_ENUM_REFERENCE_STOP: u8 = 2; +pub(crate) const INVALID_CONSTANT_REFERENCE_STOP: u8 = 3; +/// Macro calls start with this token; SQL without it has nothing to expand. +pub(crate) const MACRO_TOKEN: char = '@'; +/// The keys whose `false` value opts a model out of SQL analysis. +pub(crate) const OPT_OUT_KEYS: [&str; 2] = ["sql_analysis", "sql_validation"]; +pub(crate) const PATH_SEPARATOR: char = '/'; +pub(crate) const CURRENT_DIRECTORY: &str = "."; +pub(crate) const NEWLINE: char = '\n'; diff --git a/crates/sqlbuild-analysis/src/compiled_project/errors.rs b/crates/sqlbuild-analysis/src/compiled_project/errors.rs new file mode 100644 index 0000000000..4e1b16bad1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/errors.rs @@ -0,0 +1,26 @@ +//! Why an enum or constant reference in a model could not be expanded. + +use crate::compiled_project::models::InaccessibleDeclaration; + +/// Why an enum or constant reference could not be expanded. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DeclarationError { + /// `kind` is `enum` or `constant`; `visible` names every declaration of that kind in scope. + Unknown { + kind: String, + name: String, + visible: Vec, + }, + UnknownMember { + name: String, + member: String, + members: Vec, + }, + Inaccessible { + kind: String, + name: String, + record: InaccessibleDeclaration, + }, + /// The native scan stopped with this code. + ScanStop(u8), +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/analysis_gate.rs b/crates/sqlbuild-analysis/src/compiled_project/main/analysis_gate.rs new file mode 100644 index 0000000000..ac89509e48 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/analysis_gate.rs @@ -0,0 +1,9 @@ +use crate::compiled_project::models::{AnalysisGate, AnalysisOverride, ModelLoopSettings}; + +/// Whether the model's SQL is analysed under the project switches and its override. +pub fn analysis_gate( + settings: ModelLoopSettings, + analysis_override: AnalysisOverride, +) -> AnalysisGate { + crate::compiled_project::_helpers::plain::analysis_gate(settings, analysis_override) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/authored_model_offset.rs b/crates/sqlbuild-analysis/src/compiled_project/main/authored_model_offset.rs new file mode 100644 index 0000000000..c355564167 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/authored_model_offset.rs @@ -0,0 +1,36 @@ +//! `_authored_model_location`: an offset of expanded model SQL in the authored model file. + +use sqlbuild_core::text::main::is_python_space::is_python_space; + +use crate::compiled_project::_helpers::locating::{map_through_passes, model_query_start}; +use crate::compiled_project::models::SplicedSpan; + +/// The authored file offset of `offset` in `sql`, when expansion did not generate it. +pub fn authored_model_offset( + contents: &str, + query_sql: &str, + expanded: (&str, usize), + passes: &[Vec], +) -> Option { + let (sql, offset) = expanded; + let characters: Vec = contents.chars().collect(); + let body_start: usize = model_query_start(contents, query_sql).or_else(|| { + let leading: usize = characters + .iter() + .take_while(|character| is_python_space(**character)) + .count(); + let query: Vec = query_sql.chars().collect(); + characters[leading..].starts_with(&query).then_some(leading) + })?; + let sql_characters: Vec = sql.chars().collect(); + let (mapped, generated) = map_through_passes(offset, passes); + let authored: usize = body_start + mapped; + if generated + || offset >= sql_characters.len() + || authored >= characters.len() + || characters[authored] != sql_characters[offset] + { + return None; + } + Some(authored) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/calls_macros.rs b/crates/sqlbuild-analysis/src/compiled_project/main/calls_macros.rs new file mode 100644 index 0000000000..5eccf875d1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/calls_macros.rs @@ -0,0 +1,4 @@ +/// Whether declaration-expanded SQL still calls macros, which the attachment expands. +pub fn calls_macros(sql: &str) -> bool { + crate::compiled_project::_helpers::plain::calls_macros(sql) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/declaration_error_message.rs b/crates/sqlbuild-analysis/src/compiled_project/main/declaration_error_message.rs new file mode 100644 index 0000000000..b9b3f4e395 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/declaration_error_message.rs @@ -0,0 +1,22 @@ +use crate::compiled_project::_helpers::declaration_errors; +use crate::compiled_project::errors::DeclarationError; + +/// The message the attachment raises for one enum or constant expansion error. +pub fn declaration_error_message(error: &DeclarationError, consumer: &str) -> String { + match error { + DeclarationError::Unknown { + kind, + name, + visible, + } => declaration_errors::unknown_declaration(kind, name, consumer, visible), + DeclarationError::UnknownMember { + name, + member, + members, + } => declaration_errors::unknown_member(name, member, consumer, members), + DeclarationError::Inaccessible { kind, name, record } => { + declaration_errors::inaccessible(kind, name, consumer, record) + } + DeclarationError::ScanStop(code) => declaration_errors::scan_stop(*code, consumer), + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/enum_member_literal.rs b/crates/sqlbuild-analysis/src/compiled_project/main/enum_member_literal.rs new file mode 100644 index 0000000000..f03562cda0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/enum_member_literal.rs @@ -0,0 +1,4 @@ +/// An enum member's SQL literal: integers as written, text single-quoted with quotes doubled. +pub fn enum_member_literal(text: &str, integer: bool) -> String { + crate::compiled_project::_helpers::splice::enum_member_literal(text, integer) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/header_metadata_keys.rs b/crates/sqlbuild-analysis/src/compiled_project/main/header_metadata_keys.rs new file mode 100644 index 0000000000..122e470e0f --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/header_metadata_keys.rs @@ -0,0 +1,6 @@ +use crate::compiled_project::constants::HEADER_METADATA_KEYS; + +/// Whether `key` is header metadata, attached as schema and stripped from the model's config. +pub fn is_header_metadata_key(key: &str) -> bool { + HEADER_METADATA_KEYS.contains(&key) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/legacy_schema_error.rs b/crates/sqlbuild-analysis/src/compiled_project/main/legacy_schema_error.rs new file mode 100644 index 0000000000..58f235366d --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/legacy_schema_error.rs @@ -0,0 +1,9 @@ +use crate::compiled_project::models::LegacySchemaFile; + +/// The error for a model `(name, relative path)` that a legacy schema file still declares. +pub fn legacy_schema_error( + model: (&str, &str), + schema_files: &[LegacySchemaFile], +) -> Option { + crate::compiled_project::_helpers::sources::legacy_schema_error(model, schema_files) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/located_scan_error.rs b/crates/sqlbuild-analysis/src/compiled_project/main/located_scan_error.rs new file mode 100644 index 0000000000..fffe239edf --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/located_scan_error.rs @@ -0,0 +1,14 @@ +use sqlbuild_sqltext::sql_references::models::ReferenceScanFailure; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::compiled_project::models::ScanFaultOrigin; + +/// The message a model's reference-scan fault raises, located in authored text when it can be. +pub fn located_scan_error( + failure: &ReferenceScanFailure, + sql: &str, + origin: &ScanFaultOrigin<'_>, + syntax: &LexicalSyntax, +) -> String { + crate::compiled_project::_helpers::locating::located_scan_error(failure, sql, origin, syntax) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/merge_call_site_references.rs b/crates/sqlbuild-analysis/src/compiled_project/main/merge_call_site_references.rs new file mode 100644 index 0000000000..8ef6434166 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/merge_call_site_references.rs @@ -0,0 +1,4 @@ +/// The expanded SQL's references, then each macro argument reference once; unchanged without any. +pub fn merge_call_site_references(references: Vec, arguments: Vec) -> Vec { + crate::compiled_project::_helpers::splice::merge_call_site_references(references, arguments) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs b/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs new file mode 100644 index 0000000000..a6c274ad63 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/mod.rs @@ -0,0 +1,17 @@ +pub mod analysis_gate; +pub mod authored_model_offset; +pub mod calls_macros; +pub mod declaration_error_message; +pub mod enum_member_literal; +pub mod header_metadata_keys; +pub mod legacy_schema_error; +pub mod located_scan_error; +pub mod merge_call_site_references; +pub mod model_facts; +pub mod opt_out_position; +pub mod plain_model_facts; +pub mod project_models; +pub mod record_model; +pub mod record_model_analysis; +pub mod splice_declarations; +pub mod text_position; diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/model_facts.rs b/crates/sqlbuild-analysis/src/compiled_project/main/model_facts.rs new file mode 100644 index 0000000000..519587b8d5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/model_facts.rs @@ -0,0 +1,9 @@ +use crate::compiled_project::models::{CompiledModelFacts, CompiledProjectFacts}; + +/// The facts recorded for the model named `name`. +pub fn model_facts<'project>( + project: &'project CompiledProjectFacts, + name: &str, +) -> Option<&'project CompiledModelFacts> { + project.model(name) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/opt_out_position.rs b/crates/sqlbuild-analysis/src/compiled_project/main/opt_out_position.rs new file mode 100644 index 0000000000..f4e691ff5f --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/opt_out_position.rs @@ -0,0 +1,8 @@ +use sqlbuild_core::text::models::PythonText; + +use crate::compiled_project::models::TextPosition; + +/// Where the first whole-word `sql_analysis` or `sql_validation` at or after character `start` is. +pub fn opt_out_position(python: PythonText, text: &str, start: usize) -> TextPosition { + crate::compiled_project::_helpers::sources::opt_out_position(python, text, start) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/plain_model_facts.rs b/crates/sqlbuild-analysis/src/compiled_project/main/plain_model_facts.rs new file mode 100644 index 0000000000..368f8c6c3e --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/plain_model_facts.rs @@ -0,0 +1,6 @@ +use crate::compiled_project::models::{CompiledModelFacts, PlainModel}; + +/// A natively finished model's retained facts. +pub fn plain_model_facts(model: PlainModel) -> CompiledModelFacts { + crate::compiled_project::_helpers::plain::plain_model_facts(model) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/project_models.rs b/crates/sqlbuild-analysis/src/compiled_project/main/project_models.rs new file mode 100644 index 0000000000..fc00acbd95 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/project_models.rs @@ -0,0 +1,6 @@ +use crate::compiled_project::models::{CompiledModelFacts, CompiledProjectFacts}; + +/// Every recorded model's facts, in production order. +pub fn project_models(project: &CompiledProjectFacts) -> &[CompiledModelFacts] { + &project.models +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/record_model.rs b/crates/sqlbuild-analysis/src/compiled_project/main/record_model.rs new file mode 100644 index 0000000000..7567fb65f5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/record_model.rs @@ -0,0 +1,6 @@ +use crate::compiled_project::models::{CompiledModelFacts, CompiledProjectFacts}; + +/// Retain one model's compile-input facts; recording a name again replaces its facts in place. +pub fn record_model(project: &mut CompiledProjectFacts, model: CompiledModelFacts) { + project.record_model(model); +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/record_model_analysis.rs b/crates/sqlbuild-analysis/src/compiled_project/main/record_model_analysis.rs new file mode 100644 index 0000000000..7b43478df6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/record_model_analysis.rs @@ -0,0 +1,10 @@ +use crate::compiled_project::models::{CompiledProjectFacts, ModelAnalysisFact}; + +/// Attach a recorded model's typed analysis result; `false` when the model was never recorded. +pub fn record_model_analysis( + project: &mut CompiledProjectFacts, + name: &str, + analysis: ModelAnalysisFact, +) -> bool { + project.record_analysis(name, analysis) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/splice_declarations.rs b/crates/sqlbuild-analysis/src/compiled_project/main/splice_declarations.rs new file mode 100644 index 0000000000..93ebdbaf38 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/splice_declarations.rs @@ -0,0 +1,6 @@ +use crate::compiled_project::models::{DeclarationSplice, SplicedSql}; + +/// `sql` with each scanned reference, in source order, replaced by its rendered text. +pub fn splice_declarations(sql: &str, splices: &[DeclarationSplice]) -> SplicedSql { + crate::compiled_project::_helpers::splice::splice_declarations(sql, splices) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/main/text_position.rs b/crates/sqlbuild-analysis/src/compiled_project/main/text_position.rs new file mode 100644 index 0000000000..bd53e654dd --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/main/text_position.rs @@ -0,0 +1,8 @@ +use crate::compiled_project::_helpers::sources::char_position; +use crate::compiled_project::models::TextPosition; + +/// The one-based line and column of character `offset` in `text`. +pub fn text_position(text: &str, offset: usize) -> TextPosition { + let characters: Vec = text.chars().collect(); + char_position(&characters, offset) +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/mod.rs b/crates/sqlbuild-analysis/src/compiled_project/mod.rs new file mode 100644 index 0000000000..eec94b5503 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/mod.rs @@ -0,0 +1,10 @@ +//! The compile's natively owned project: facts each producer retains for every consumer. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/compiled_project/models.rs b/crates/sqlbuild-analysis/src/compiled_project/models.rs new file mode 100644 index 0000000000..a2f5abe0cc --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/models.rs @@ -0,0 +1,245 @@ +//! Facts a compile's producers retain about each resource, in production order. + +use std::collections::{BTreeMap, HashMap}; + +use serde_json::Value; + +/// One compile's project facts; models keep the order the compile-input stage produced them. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CompiledProjectFacts { + pub(crate) models: Vec, + pub(crate) model_index: HashMap, +} + +/// What the compile-input stage produced for one model, plus its analysis once transferred. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CompiledModelFacts { + pub name: String, + pub relative_path: String, + /// SQL after variable, declaration and macro expansion. + pub query_sql: String, + /// The authored query below the `MODEL (...)` header. + pub authored_query_sql: String, + /// The whole authored file. + pub authored_sql: String, + pub config: ModelConfigFacts, + pub references: Vec, + pub schema: Option, + pub enum_columns: Vec, + pub enum_declarations: Vec, + pub constant_declarations: Vec, + /// The typed analysis result; `None` until the assembly stage transfers it. + pub analysis: Option, +} + +/// A model's effective config values with sorted keys, as the rules request has always read them. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ModelConfigFacts { + pub values: BTreeMap, + /// Why the values could not be encoded, raised only when a consumer reads them. + pub encoding_error: Option, + pub header_keys: Vec, + pub layer_schema: Option, + /// The `contract` value when it is text. + pub contract: Option, + /// The `materialized` value when it is text; `table` when it is absent. + pub materialized: Option, + /// The `incremental_mode` value when it is text. + pub incremental_mode: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelReferenceFact { + pub kind: String, + pub name: String, + pub package: Option, +} + +/// A model's declared schema entry. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelSchemaFact { + /// Audits declared on the entry itself, not on its columns. + pub audit_count: u32, + pub columns: Vec, + pub dynamic_columns: Vec, + /// Whether a column or family declares a type, so the contract checks types. + pub type_enforcement: bool, + /// Whether the entry comes from a reusable named schema. + pub named_schema: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SchemaColumnFact { + pub name: String, + pub data_type: Option, + pub nullable: Option, + pub audit_count: u32, + /// Whether the column is declared outside the model file, in a named schema. + pub declared_outside_model: bool, +} + +/// One declared dynamic-column family; `key` is its name casefolded. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DynamicFamilyFact { + pub key: String, + pub name: String, + pub pivot_column: String, + pub value_column: String, + pub aggregate: String, + pub data_type: String, + pub name_pattern: Option, +} + +/// An enum or constant declaration a model can see. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct DeclarationFact { + pub name: String, + pub relative_path: String, + pub members: Vec<(String, Value)>, + pub value: Option, + pub value_type: Option, + pub render_as: Option, +} + +/// A model's typed analysis outcome: inferred output columns and its dynamic pivot proof. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelAnalysisFact { + /// Each inferred output column's name and type, in output order. + pub inferred_columns: Vec<(String, Option)>, + pub dynamic_proof: Option, + /// Whether analysis inferred output columns at all. + pub inferred: bool, + /// Whether each inferred output column may be null, in output order. + pub nullable_columns: Vec, + pub fast_lineage_has_star: bool, + pub unchecked_output_columns: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DynamicProofFact { + pub output_proven: bool, + pub bare_dynamic_pivot: bool, + /// `(family key, inferred type)` per declared family. + pub families: Vec<(String, Option)>, + pub failure_reason: Option, + /// `(family name, inferred type)` per declared family, as authored. + pub named_families: Vec<(String, Option)>, +} + +/// The project-wide switches the model loop reads. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ModelLoopSettings { + pub sql_analysis: bool, + pub no_sql_validation: bool, + pub defer_sql_validation: bool, + pub require_sql_analysis: bool, +} + +/// The `description` config value's shape. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum DescriptionShape { + #[default] + Absent, + Text(String), + /// Present but not a non-blank string. + Invalid, +} + +/// The per-model `sql_analysis` / legacy `sql_validation` override. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AnalysisOverride { + #[default] + Inherit, + Enabled, + Disabled, + /// Both keys are set to different values. + Conflict, +} + +/// Whether a model's SQL is analysed, after the project switches and its override. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AnalysisGate { + Enabled, + Disabled, + /// The override keys disagree, which the attachment reports as P003. + Conflict, +} + +/// A known declaration the consumer cannot see, as its scope record describes it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct InaccessibleDeclaration { + pub path: String, + pub line: u64, + pub column: u64, + pub scope: String, + pub owner: String, + /// `(kind, name)` of the owning resource, when the declaration is resource-owned. + pub owner_identity: Option<(String, String)>, +} + +/// One scanned declaration reference's character range and the text that replaces it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DeclarationSplice { + pub start: usize, + pub end: usize, + pub replacement: String, +} + +/// A substituted region: its source and output character ranges. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SplicedSpan { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} + +/// SQL with declaration references replaced, and where each replacement landed. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SplicedSql { + pub sql: String, + pub spans: Vec, +} + +/// A natively finished model's facts. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PlainModel { + pub name: String, + pub relative_path: String, + pub query_sql: String, + pub authored_query_sql: String, + pub authored_sql: String, + pub config: ModelConfigFacts, + /// The attached schema entry; present when the header declares description, columns or audits. + pub schema: Option, + pub references: Vec, + /// Header columns typed as an enum, in column order. + pub enum_columns: Vec, + pub enum_declarations: Vec, + pub constant_declarations: Vec, +} + +/// A one-based line and column, counted in characters. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct TextPosition { + pub line: usize, + pub column: usize, +} + +/// A legacy schema file that declares model entries, by its relative path. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LegacySchemaFile { + pub relative_path: String, + pub model_names: Vec, +} + +/// The authored model a reference-scan fault is located in. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ScanFaultOrigin<'a> { + /// The POSIX relative path the error names. + pub relative_path: &'a str, + pub contents: &'a str, + pub query_sql: &'a str, + /// The declaration and macro expansion passes, or `None` when the SQL was not authored text. + pub passes: Option>>, +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests.rs b/crates/sqlbuild-analysis/src/compiled_project/tests.rs new file mode 100644 index 0000000000..09a35bb354 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/tests.rs @@ -0,0 +1,16 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_declaration_errors.rs"] +mod test_declaration_errors; +#[path = "tests/test_locating.rs"] +mod test_locating; +#[path = "tests/test_plain.rs"] +mod test_plain; +#[path = "tests/test_sources.rs"] +mod test_sources; +#[path = "tests/test_splice.rs"] +mod test_splice; +#[path = "tests/test_store.rs"] +mod test_store; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs new file mode 100644 index 0000000000..d01e4ad307 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs @@ -0,0 +1,79 @@ +use crate::compiled_project::models::{CompiledModelFacts, ModelAnalysisFact}; + +pub(super) fn model(name: &str, query_sql: &str) -> CompiledModelFacts { + CompiledModelFacts { + name: name.to_owned(), + query_sql: query_sql.to_owned(), + ..CompiledModelFacts::default() + } +} + +pub(super) fn analysis(column: &str) -> ModelAnalysisFact { + ModelAnalysisFact { + inferred_columns: vec![(column.to_owned(), Some("INTEGER".to_owned()))], + dynamic_proof: None, + ..ModelAnalysisFact::default() + } +} + +pub(super) fn splice( + start: usize, + end: usize, + replacement: &str, +) -> crate::compiled_project::models::DeclarationSplice { + crate::compiled_project::models::DeclarationSplice { + start, + end, + replacement: replacement.to_owned(), + } +} + +pub(super) fn analysing_settings( + sql_analysis: bool, +) -> crate::compiled_project::models::ModelLoopSettings { + crate::compiled_project::models::ModelLoopSettings { + sql_analysis, + defer_sql_validation: true, + ..crate::compiled_project::models::ModelLoopSettings::default() + } +} + +pub(super) fn python_312() -> sqlbuild_core::text::models::PythonText { + sqlbuild_core::text::main::python_text::python_text((3, 12), "15.0.0") + .expect("Python 3.12 is supported") +} + +pub(super) fn legacy_schema_files( + files: &[(&str, &[&str])], +) -> Vec { + files + .iter() + .map( + |(path, names)| crate::compiled_project::models::LegacySchemaFile { + relative_path: (*path).to_owned(), + model_names: names.iter().map(|name| (*name).to_owned()).collect(), + }, + ) + .collect() +} + +pub(super) fn generic_syntax() -> sqlbuild_sqltext::sql_scan::models::LexicalSyntax { + sqlbuild_sqltext::sql_scan::models::LexicalSyntax { + line_comment_prefixes: vec!["--".to_owned()], + ..sqlbuild_sqltext::sql_scan::models::LexicalSyntax::default() + } +} + +pub(super) fn span( + source_start: usize, + source_end: usize, + output_start: usize, + output_end: usize, +) -> crate::compiled_project::models::SplicedSpan { + crate::compiled_project::models::SplicedSpan { + source_start, + source_end, + output_start, + output_end, + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_declaration_errors.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_declaration_errors.rs new file mode 100644 index 0000000000..fa83f0a5ab --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_declaration_errors.rs @@ -0,0 +1,67 @@ +use crate::compiled_project::errors::DeclarationError; +use crate::compiled_project::main::declaration_error_message::declaration_error_message; +use crate::compiled_project::models::InaccessibleDeclaration; +use crate::compiled_project::tests::test_types::DeclarationErrorTestCase; + +#[test] +fn given_declaration_error_when_wording_then_attachment_message_is_returned() { + let test_cases = [ + DeclarationErrorTestCase { + description: "unknown private enum lists visible enums sorted", + error: DeclarationError::Unknown { + kind: "enum".to_owned(), + name: "_status".to_owned(), + visible: vec!["region".to_owned(), "channel".to_owned()], + }, + expected_message: "Unknown enum '_status' in this model in 'models/orders.sql'. \ + Visible enums: channel, region", + }, + DeclarationErrorTestCase { + description: "unknown constant with none visible", + error: DeclarationError::Unknown { + kind: "constant".to_owned(), + name: "cap".to_owned(), + visible: Vec::new(), + }, + expected_message: "Unknown constant 'cap' in 'models/orders.sql'. Visible constants: none", + }, + DeclarationErrorTestCase { + description: "inaccessible resource-owned constant", + error: DeclarationError::Inaccessible { + kind: "constant".to_owned(), + name: "cap".to_owned(), + record: InaccessibleDeclaration { + path: "models/marts/_constants/limits.sql".to_owned(), + line: 1, + column: 11, + scope: "directory".to_owned(), + owner: "models/marts".to_owned(), + owner_identity: Some(("model".to_owned(), "orders_daily".to_owned())), + }, + }, + expected_message: "Constant 'cap' is known but inaccessible in 'models/orders.sql'. \ + It is defined at models/marts/_constants/limits.sql:1:11 with \ + directory scope owned by 'models/marts' (model 'orders_daily'); \ + consumer path: 'models/orders.sql'", + }, + DeclarationErrorTestCase { + description: "invalid enum reference stop", + error: DeclarationError::ScanStop(2), + expected_message: "Invalid enum reference in 'models/orders.sql'", + }, + DeclarationErrorTestCase { + description: "unclosed quote stop", + error: DeclarationError::ScanStop(0), + expected_message: "Enum and constant expansion contains an unclosed quoted string", + }, + ]; + + for test_case in test_cases { + assert_eq!( + declaration_error_message(&test_case.error, "models/orders.sql"), + test_case.expected_message, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_locating.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_locating.rs new file mode 100644 index 0000000000..54f612c3b7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_locating.rs @@ -0,0 +1,79 @@ +use crate::compiled_project::main::located_scan_error::located_scan_error; +use crate::compiled_project::models::ScanFaultOrigin; +use crate::compiled_project::tests::helpers::{generic_syntax, span}; +use crate::compiled_project::tests::test_types::LocatedScanErrorTestCase; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; + +#[test] +fn given_reference_scan_fault_when_locating_then_python_message_is_returned() { + let test_cases = [ + LocatedScanErrorTestCase { + description: "unclosed call after the header", + sql: "SELECT *\nFROM __ref(\"orders\"", + contents: "MODEL (description \"Orders\");\n\nSELECT *\nFROM __ref(\"orders\"", + passes: Some(vec![Vec::new(), Vec::new()]), + expected_message: "models/orders.sql:4:6: SQL reference contains an unclosed parenthesis", + }, + LocatedScanErrorTestCase { + description: "unclosed quote counted in code points", + sql: "SELECT 'é', __ref(\"orders\") WHERE note = 'open", + contents: "MODEL ();\nSELECT 'é', __ref(\"orders\") WHERE note = 'open", + passes: Some(vec![Vec::new(), Vec::new()]), + expected_message: "models/orders.sql:2:42: SQL reference contains an unclosed quoted string", + }, + LocatedScanErrorTestCase { + description: "fault after neutral expansions maps through both passes", + sql: "SELECT 'open', 1 + 2\nFROM __ref(\"orders\"", + contents: "MODEL ();\nSELECT @enum(\"s\").OPEN, @m()\nFROM __ref(\"orders\"", + passes: Some(vec![vec![span(7, 22, 7, 13)], vec![span(15, 19, 15, 20)]]), + expected_message: "models/orders.sql:3:6: SQL reference contains an unclosed parenthesis", + }, + LocatedScanErrorTestCase { + description: "fault inside macro output is unlocated", + sql: "SELECT * FROM __ref(\"orders\") /* expanded", + contents: "MODEL ();\nSELECT * FROM __ref(\"orders\") @expand()", + passes: Some(vec![Vec::new(), vec![span(30, 39, 30, 41)]]), + expected_message: "models/orders.sql: SQL reference contains an unclosed block comment", + }, + LocatedScanErrorTestCase { + description: "authored fault shaped by an unclosed macro quote is unlocated", + sql: "SELECT 'x || ' || note || 'open", + contents: "MODEL ();\nSELECT @q() || ' || note || 'open", + passes: Some(vec![Vec::new(), vec![span(7, 11, 7, 9)]]), + expected_message: "models/orders.sql: SQL reference contains an unclosed quoted string", + }, + LocatedScanErrorTestCase { + description: "without authored passes only the file is named", + sql: "SELECT *\nFROM __ref(\"orders\"", + contents: "MODEL ();\nSELECT *\nFROM __ref(\"orders\"", + passes: None, + expected_message: "models/orders.sql: SQL reference contains an unclosed parenthesis", + }, + ]; + + for test_case in test_cases { + let ReferenceExtraction::Failed(failure) = + extract_sql_references(test_case.sql, &generic_syntax()) + else { + panic!("{}: the scan must fail", test_case.description); + }; + let query_sql: String = test_case + .contents + .split_once(";\n") + .map_or(test_case.contents, |(_, query)| query.trim_start()) + .to_owned(); + let origin = ScanFaultOrigin { + relative_path: "models/orders.sql", + contents: test_case.contents, + query_sql: &query_sql, + passes: test_case.passes, + }; + assert_eq!( + located_scan_error(&failure, test_case.sql, &origin, &generic_syntax()), + test_case.expected_message, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_plain.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_plain.rs new file mode 100644 index 0000000000..85bef370fa --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_plain.rs @@ -0,0 +1,46 @@ +use crate::compiled_project::main::analysis_gate::analysis_gate; +use crate::compiled_project::models::{AnalysisGate, AnalysisOverride}; +use crate::compiled_project::tests::helpers::analysing_settings; +use crate::compiled_project::tests::test_types::AnalysisGateTestCase; + +#[test] +fn given_override_when_gating_analysis_then_project_switches_come_first() { + let test_cases = [ + AnalysisGateTestCase { + description: "inherited", + sql_analysis: true, + analysis_override: AnalysisOverride::Inherit, + expected_gate: AnalysisGate::Enabled, + }, + AnalysisGateTestCase { + description: "opted out", + sql_analysis: true, + analysis_override: AnalysisOverride::Disabled, + expected_gate: AnalysisGate::Disabled, + }, + AnalysisGateTestCase { + description: "conflicting keys", + sql_analysis: true, + analysis_override: AnalysisOverride::Conflict, + expected_gate: AnalysisGate::Conflict, + }, + AnalysisGateTestCase { + description: "project disables analysis before the conflict is read", + sql_analysis: false, + analysis_override: AnalysisOverride::Conflict, + expected_gate: AnalysisGate::Disabled, + }, + ]; + + for test_case in test_cases { + assert_eq!( + analysis_gate( + analysing_settings(test_case.sql_analysis), + test_case.analysis_override + ), + test_case.expected_gate, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_sources.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_sources.rs new file mode 100644 index 0000000000..0ea1c95261 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_sources.rs @@ -0,0 +1,110 @@ +use crate::compiled_project::main::legacy_schema_error::legacy_schema_error; +use crate::compiled_project::main::opt_out_position::opt_out_position; +use crate::compiled_project::main::text_position::text_position; +use crate::compiled_project::models::TextPosition; +use crate::compiled_project::tests::helpers::{legacy_schema_files, python_312}; +use crate::compiled_project::tests::test_types::{LegacySchemaTestCase, OptOutPositionTestCase}; + +#[test] +fn given_text_when_locating_opt_out_then_first_whole_word_key_after_start_is_found() { + let test_cases = [ + OptOutPositionTestCase { + description: "header key on the first line", + text: "MODEL (sql_analysis false);", + start: 0, + expected_position: TextPosition { line: 1, column: 8 }, + }, + OptOutPositionTestCase { + description: "legacy key after a longer identifier", + text: "[path_defaults.marts]\nmy_sql_analysis = 1\nsql_validation = false\n", + start: 6, + expected_position: TextPosition { line: 3, column: 1 }, + }, + OptOutPositionTestCase { + description: "no key falls back to the start of the file", + text: "é\nmaterialized = \"view\"", + start: 0, + expected_position: TextPosition { line: 1, column: 1 }, + }, + ]; + + for test_case in test_cases { + assert_eq!( + opt_out_position(python_312(), test_case.text, test_case.start), + test_case.expected_position, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_legacy_schema_files_when_matching_model_then_attachment_error_is_returned() { + let test_cases = [ + LegacySchemaTestCase { + description: "schema file in another directory", + schema_files: &[("models/staging/schema.yml", &["orders"])], + expected_error: None, + }, + LegacySchemaTestCase { + description: "root schema file declares the model", + schema_files: &[("schema.yml", &["orders"])], + expected_error: Some( + "Schema file schema.yml declares model 'orders', but model metadata must live in \ + models/marts/orders.sql MODEL(...). Move description, columns, and audits into \ + the model header.", + ), + }, + LegacySchemaTestCase { + description: "two schema files declare the model", + schema_files: &[ + ("models/schema.yml", &["orders"]), + ("models/marts/schema.yml", &["orders", "customers"]), + ], + expected_error: Some( + "Model file models/marts/orders.sql matched multiple schema.yml declarations: \ + models/schema.yml, models/marts/schema.yml", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + legacy_schema_error( + ("orders", "models/marts/orders.sql"), + &legacy_schema_files(test_case.schema_files) + ) + .as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_character_offset_when_positioning_then_line_and_column_count_characters() { + let test_cases = [ + OptOutPositionTestCase { + description: "start of text", + text: "SELECT 1", + start: 0, + expected_position: TextPosition { line: 1, column: 1 }, + }, + OptOutPositionTestCase { + description: "after a multi-byte character on the second line", + text: "MODEL ();\né __ref", + start: 12, + expected_position: TextPosition { line: 2, column: 3 }, + }, + ]; + + for test_case in test_cases { + assert_eq!( + text_position(test_case.text, test_case.start), + test_case.expected_position, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_splice.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_splice.rs new file mode 100644 index 0000000000..9e74424e86 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_splice.rs @@ -0,0 +1,101 @@ +use crate::compiled_project::main::enum_member_literal::enum_member_literal; +use crate::compiled_project::main::merge_call_site_references::merge_call_site_references; +use crate::compiled_project::main::splice_declarations::splice_declarations; +use crate::compiled_project::tests::helpers::splice; +use crate::compiled_project::tests::test_types::{ + EnumLiteralTestCase, MergeReferencesTestCase, SpliceTestCase, +}; + +#[test] +fn given_scanned_references_when_splicing_then_character_ranges_are_kept() { + let test_cases = [ + SpliceTestCase { + description: "two references", + sql: "SELECT @enum.status.open, @const.limit", + splices: vec![splice(7, 24, "'open'"), splice(26, 38, "10")], + expected_sql: "SELECT 'open', 10", + expected_output_ranges: vec![(7, 13), (15, 17)], + }, + SpliceTestCase { + description: "multi-byte text before a reference", + sql: "SELECT 'é', @const.limit", + splices: vec![splice(12, 24, "'ü'")], + expected_sql: "SELECT 'é', 'ü'", + expected_output_ranges: vec![(12, 15)], + }, + ]; + + for test_case in test_cases { + let spliced = splice_declarations(test_case.sql, &test_case.splices); + let ranges: Vec<(usize, usize)> = spliced + .spans + .iter() + .map(|span| (span.output_start, span.output_end)) + .collect(); + + assert_eq!( + spliced.sql, test_case.expected_sql, + "{}", + test_case.description + ); + assert_eq!( + ranges, test_case.expected_output_ranges, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_enum_member_when_rendering_then_text_is_quoted_and_integers_are_bare() { + let test_cases = [ + EnumLiteralTestCase { + description: "text member", + text: "it's", + integer: false, + expected_literal: "'it''s'", + }, + EnumLiteralTestCase { + description: "integer member", + text: "42", + integer: true, + expected_literal: "42", + }, + ]; + + for test_case in test_cases { + assert_eq!( + enum_member_literal(test_case.text, test_case.integer), + test_case.expected_literal, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_macro_argument_references_when_merging_then_each_reference_appears_once() { + let test_cases = [ + MergeReferencesTestCase { + description: "no arguments keep scanned duplicates", + references: vec!["orders", "orders"], + arguments: vec![], + expected_references: vec!["orders", "orders"], + }, + MergeReferencesTestCase { + description: "arguments deduplicate the whole list in first-seen order", + references: vec!["orders", "orders", "customers"], + arguments: vec!["payments", "customers"], + expected_references: vec!["orders", "customers", "payments"], + }, + ]; + + for test_case in test_cases { + assert_eq!( + merge_call_site_references(test_case.references, test_case.arguments), + test_case.expected_references, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs new file mode 100644 index 0000000000..d3041bb078 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs @@ -0,0 +1,69 @@ +use crate::compiled_project::main::model_facts::model_facts; +use crate::compiled_project::main::record_model::record_model; +use crate::compiled_project::main::record_model_analysis::record_model_analysis; +use crate::compiled_project::models::CompiledProjectFacts; +use crate::compiled_project::tests::helpers::{analysis, model}; +use crate::compiled_project::tests::test_types::RecordTestCase; + +#[test] +fn given_recorded_models_when_reading_then_order_and_latest_facts_are_kept() { + let test_cases = [ + RecordTestCase { + description: "production order is kept", + recorded: vec![model("orders", "SELECT 1"), model("customers", "SELECT 2")], + analysed: "orders", + expected_order: &["orders", "customers"], + expected_query_sql: "SELECT 1", + expected_analysis_recorded: true, + }, + RecordTestCase { + description: "recording a model again replaces it in place", + recorded: vec![ + model("orders", "SELECT 1"), + model("customers", "SELECT 2"), + model("orders", "SELECT 3"), + ], + analysed: "orders", + expected_order: &["orders", "customers"], + expected_query_sql: "SELECT 3", + expected_analysis_recorded: true, + }, + RecordTestCase { + description: "analysis for an unrecorded model is refused", + recorded: vec![model("orders", "SELECT 1")], + analysed: "payments", + expected_order: &["orders"], + expected_query_sql: "SELECT 1", + expected_analysis_recorded: false, + }, + ]; + + for test_case in test_cases { + let mut project: CompiledProjectFacts = CompiledProjectFacts::default(); + for recorded in test_case.recorded { + record_model(&mut project, recorded); + } + let recorded_analysis: bool = + record_model_analysis(&mut project, test_case.analysed, analysis("order_id")); + let order: Vec<&str> = project + .models + .iter() + .map(|facts| facts.name.as_str()) + .collect(); + let query_sql: Option<&str> = + model_facts(&project, "orders").map(|facts| facts.query_sql.as_str()); + + assert_eq!(order, test_case.expected_order, "{}", test_case.description); + assert_eq!( + query_sql, + Some(test_case.expected_query_sql), + "{}", + test_case.description + ); + assert_eq!( + recorded_analysis, test_case.expected_analysis_recorded, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_types.rs new file mode 100644 index 0000000000..5f9946a4aa --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_types.rs @@ -0,0 +1,67 @@ +use crate::compiled_project::models::CompiledModelFacts; + +pub(super) struct RecordTestCase { + pub(super) description: &'static str, + pub(super) recorded: Vec, + pub(super) analysed: &'static str, + pub(super) expected_order: &'static [&'static str], + pub(super) expected_query_sql: &'static str, + pub(super) expected_analysis_recorded: bool, +} + +pub(super) struct SpliceTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) splices: Vec, + pub(super) expected_sql: &'static str, + pub(super) expected_output_ranges: Vec<(usize, usize)>, +} + +pub(super) struct EnumLiteralTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) integer: bool, + pub(super) expected_literal: &'static str, +} + +pub(super) struct MergeReferencesTestCase { + pub(super) description: &'static str, + pub(super) references: Vec<&'static str>, + pub(super) arguments: Vec<&'static str>, + pub(super) expected_references: Vec<&'static str>, +} + +pub(super) struct AnalysisGateTestCase { + pub(super) description: &'static str, + pub(super) sql_analysis: bool, + pub(super) analysis_override: crate::compiled_project::models::AnalysisOverride, + pub(super) expected_gate: crate::compiled_project::models::AnalysisGate, +} + +pub(super) struct DeclarationErrorTestCase { + pub(super) description: &'static str, + pub(super) error: crate::compiled_project::errors::DeclarationError, + pub(super) expected_message: &'static str, +} + +pub(super) struct OptOutPositionTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) start: usize, + pub(super) expected_position: crate::compiled_project::models::TextPosition, +} + +pub(super) struct LegacySchemaTestCase { + pub(super) description: &'static str, + /// `(schema file, model names)` per legacy schema file. + pub(super) schema_files: &'static [(&'static str, &'static [&'static str])], + pub(super) expected_error: Option<&'static str>, +} + +pub(super) struct LocatedScanErrorTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) contents: &'static str, + pub(super) passes: Option>>, + pub(super) expected_message: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs index 9f7007301d..392920ac81 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize}; use crate::compiler::_helpers::sql_tests::authored_macro_calls::calls_macro; use crate::compiler::_helpers::sql_tests::reference_calls::body_calls; +use crate::compiler::models::{SqlTestClassification, SqlTestCteText, SqlTestRequest}; use crate::constants::{ DIRECT_DEPENDENCY_PATH_LENGTH, MACRO_TEST_MODE, OPERAND_KEYWORDS, OPERATOR_CHARACTERS, SELECT_STAR_PROJECTION, TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE, VALUE_KEYWORDS, @@ -51,29 +52,12 @@ const RESERVED_NAMES: [&str; 5] = [ #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct BatchRequest { - tests: Vec, + tests: Vec, /// The adapter's lexical rules; generic SQL when absent. #[serde(default)] syntax: Option, } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct TestRequest { - sql: String, - file_label: String, - mode: String, - /// The test before macro and variable expansion, where macro calls are still authored. - #[serde(default)] - raw: bool, - /// Read only the CTE names and body offsets of the authored block, before any expansion. - #[serde(default)] - authored: bool, - /// The authored text is a scenario rather than a SQL test block. - #[serde(default)] - scenario: bool, -} - /// One extraction error, with help and the offending text where the author can act on it. #[derive(Debug, PartialEq, Eq)] pub(crate) struct Failure { @@ -106,51 +90,10 @@ struct ErrorResponse { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] enum BatchResponse { - Tests(Vec), + Tests(Vec), Error(ErrorResponse), } -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub(crate) struct Cte(pub(crate) String, pub(crate) String); - -#[derive(Debug, PartialEq, Eq, Serialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum Classified { - /// Each CTE's name, its code-point offset in the authored block and its stripped body. - Authored { ctes: Vec<(String, usize, String)> }, - Model { - mode: String, - authored: Vec, - #[serde(rename = "macroMocks")] - macro_mocks: Vec<(String, String)>, - #[serde(rename = "mockModels")] - mock_models: Vec, - #[serde(rename = "mockSources")] - mock_sources: Vec, - #[serde(rename = "mockSeeds")] - mock_seeds: Vec, - #[serde(rename = "mockDbtRefs")] - mock_dbt_refs: Vec, - #[serde(rename = "mockTableFunctions")] - mock_table_functions: Vec, - expected: Vec, - #[serde(rename = "expectedModels")] - expected_models: Vec, - assertions: Vec, - #[serde(rename = "assertionNames")] - assertion_names: Vec, - }, - Direct { - mode: String, - helpers: Vec, - actual: Cte, - expected: Cte, - /// Whether a helper or the expected CTE has a malformed reference call (P012). - #[serde(rename = "invalidCalls")] - invalid_calls: bool, - }, -} - pub(crate) fn extract_batch_json(request_json: &str) -> Result { let request: BatchRequest = serde_json::from_str(request_json).map_err(|error| error.to_string())?; @@ -175,6 +118,18 @@ pub(crate) fn extract_batch_json(request_json: &str) -> Result { serde_json::to_string(&BatchResponse::Tests(classified)).map_err(|error| error.to_string()) } +/// Each test's classification, or the first failure with the index of its test. +pub(crate) fn extract_tests( + tests: &[SqlTestRequest], + syntax: &LexicalSyntax, +) -> Result, (usize, Failure)> { + tests + .iter() + .enumerate() + .map(|(index, test)| extract_test(test, syntax).map_err(|failure| (index, failure))) + .collect() +} + /// Generic SQL lexical rules: `--` line comments and non-nested block comments. pub(crate) fn generic_syntax() -> LexicalSyntax { LexicalSyntax { @@ -190,11 +145,15 @@ struct TestScope<'a> { raw: bool, } -fn extract_test(test: &TestRequest, syntax: &LexicalSyntax) -> Result { +fn extract_test( + test: &SqlTestRequest, + syntax: &LexicalSyntax, +) -> Result { if test.authored { return authored_ctes(test, syntax); } - let ctes: Vec = scan_ctes(&test.sql, &test.file_label, TEST_CONTEXT, syntax)?.ctes; + let ctes: Vec = + scan_ctes(&test.sql, &test.file_label, TEST_CONTEXT, syntax)?.ctes; let scope = TestScope { file: &test.file_label, syntax, @@ -208,14 +167,17 @@ fn extract_test(test: &TestRequest, syntax: &LexicalSyntax) -> Result Result { +fn authored_ctes( + test: &SqlTestRequest, + syntax: &LexicalSyntax, +) -> Result { let context: &str = if test.scenario { SCENARIO_CONTEXT } else { TEST_CONTEXT }; match scan_ctes(&test.sql, &test.file_label, context, syntax) { - Ok(scanned) => Ok(Classified::Authored { + Ok(scanned) => Ok(SqlTestClassification::Authored { ctes: scanned .ctes .into_iter() @@ -224,7 +186,7 @@ fn authored_ctes(test: &TestRequest, syntax: &LexicalSyntax) -> Result Err(failure), - Err(_) => Ok(Classified::Authored { ctes: Vec::new() }), + Err(_) => Ok(SqlTestClassification::Authored { ctes: Vec::new() }), } } @@ -237,7 +199,7 @@ struct CteOwner<'a> { /// The top-level CTEs of one test, with the code-point offset of each stripped body. struct ScannedCtes { - ctes: Vec, + ctes: Vec, body_starts: Vec, } @@ -261,7 +223,7 @@ fn scan_ctes( if let Some(end) = consume_keyword(sql, index, "RECURSIVE") { index = skip_ignorable(sql, end, syntax)?; } - let mut ctes: Vec = Vec::new(); + let mut ctes: Vec = Vec::new(); let mut seen: HashSet = HashSet::new(); loop { let (name, end) = read_cte_name(sql, index, &owner)?; @@ -289,7 +251,7 @@ fn scan_ctes( let body = &sql[index + 1..close]; let leading = body.len() - body.trim_start_matches(is_python_space).len(); body_starts.push(sql[..index + 1 + leading].chars().count()); - ctes.push(Cte(name, python_strip(body).to_owned())); + ctes.push(SqlTestCteText(name, python_strip(body).to_owned())); index = skip_ignorable(sql, close + 1, syntax)?; if byte_at(sql, index) == Some(b',') { index = skip_ignorable(sql, index + 1, syntax)?; @@ -408,19 +370,22 @@ fn cte_name_failure(sql: &str, start: usize, end: usize, owner: &CteOwner<'_>) - } } -fn classify_model(ctes: Vec, scope: &TestScope<'_>) -> Result { +fn classify_model( + ctes: Vec, + scope: &TestScope<'_>, +) -> Result { let (file, syntax) = (scope.file, scope.syntax); validate_independence(&ctes, file, syntax)?; - let mut authored: Vec = Vec::new(); + let mut authored: Vec = Vec::new(); let mut macro_mocks: Vec<(String, String)> = Vec::new(); let mut mock_models: Vec = Vec::new(); let mut mock_sources: Vec = Vec::new(); let mut mock_seeds: Vec = Vec::new(); let mut mock_dbt_refs: Vec = Vec::new(); let mut mock_table_functions: Vec = Vec::new(); - let mut expected: Vec = Vec::new(); + let mut expected: Vec = Vec::new(); let mut expected_models: Vec = Vec::new(); - let mut assertions: Vec = Vec::new(); + let mut assertions: Vec = Vec::new(); let mut assertion_names: Vec = Vec::new(); for cte in ctes { let name = cte.0.as_str(); @@ -489,7 +454,7 @@ fn classify_model(ctes: Vec, scope: &TestScope<'_>) -> Result &'static str { } fn classify_direct( - ctes: Vec, + ctes: Vec, mode: &str, scope: &TestScope<'_>, -) -> Result { +) -> Result { let file = scope.file; let (_, actual_name, expected_name) = DIRECT_NAMES .iter() .find(|entry| entry.0 == mode) .copied() .ok_or_else(|| format!("SQL test '{file}' has unsupported mode '{mode}'"))?; - let mut helpers: Vec = Vec::new(); + let mut helpers: Vec = Vec::new(); let mut actual = None; let mut expected = None; for cte in ctes { @@ -596,7 +561,7 @@ fn classify_direct( allowed: actual_name, scope, })?; - Ok(Classified::Direct { + Ok(SqlTestClassification::Direct { mode: mode.to_owned(), helpers, actual, @@ -668,7 +633,11 @@ fn require_suffix(value: &str, label: &str, file: &str) -> Result<(), String> { } } -fn macro_mock_value(cte: &Cte, file: &str, syntax: &LexicalSyntax) -> Result { +fn macro_mock_value( + cte: &SqlTestCteText, + file: &str, + syntax: &LexicalSyntax, +) -> Result { let body = cte.1.as_str(); let mut index = skip_ignorable(body, 0, syntax)?; index = consume_keyword(body, index, "SELECT") @@ -701,7 +670,7 @@ fn macro_mock_shape(file: &str, name: &str, trailing: bool) -> String { } fn validate_expected( - cte: &Cte, + cte: &SqlTestCteText, label: &str, allow_empty_fixture: bool, scope: &TestScope<'_>, @@ -979,7 +948,7 @@ fn qualified_projection_name( } pub(crate) fn validate_independence( - ctes: &[Cte], + ctes: &[SqlTestCteText], file: &str, syntax: &LexicalSyntax, ) -> Result<(), String> { diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs index 9a2d1fcf0c..35b06efd64 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs @@ -1,6 +1,5 @@ //! `extract_sql_scenario_ctes`: the top-level scan, the independence check, then classification. -use serde::Serialize; use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ctes; use sqlbuild_scopes::relationship_names::models::{RelationshipSource, TopLevelCtes}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; @@ -8,6 +7,7 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::compiler::_helpers::sql_tests::check_independence::{ IndependenceCheck, independence_error, }; +use crate::compiler::models::SqlScenarioCtes; type Cte = (String, String); @@ -20,70 +20,33 @@ const ASSERT_PREFIX: &str = "__assert__"; const MACRO_PREFIX: &str = "__macro__"; const SCENARIO_CONTEXT: &str = "SQL scenario"; -#[derive(Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct ClassifiedScenario { - authored: Vec, - expected: Vec, - assertions: Vec, - source_fixtures: Vec, - ref_fixtures: Vec, - seed_fixtures: Vec, - dbt_ref_fixtures: Vec, - expected_models: Vec, - assertion_names: Vec, -} - -/// One scenario's error, or its classification. -#[derive(Debug, Default, Serialize)] -#[serde(rename_all = "camelCase")] -struct ScenarioOutcome { - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, - #[serde(skip_serializing_if = "Option::is_none")] - scenario: Option, -} - -/// The scenario's classification or error, as JSON; independence errors come first. -pub(crate) fn extract_scenario_json( +/// The scenario's classification or error; independence errors come first. +pub(crate) fn extract_scenario( sql: &str, file: &str, syntax: &LexicalSyntax, -) -> Result { - let outcome: ScenarioOutcome = - match scan_top_level_ctes(sql, file, RelationshipSource::Scenario, syntax) { - TopLevelCtes::Failed(message) => ScenarioOutcome { - error: Some(message), - ..ScenarioOutcome::default() - }, - TopLevelCtes::Scanned(ctes) => { - let independence: Option = independence_error( - &IndependenceCheck { - ctes: &ctes, - expected_prefix: EXPECTED_PREFIX, - assertion_prefix: ASSERT_PREFIX, - file_label: file, - context_label: SCENARIO_CONTEXT, - }, - syntax, - ); - match independence.map_or_else(|| classify(ctes, file), Err) { - Ok(scenario) => ScenarioOutcome { - scenario: Some(scenario), - ..ScenarioOutcome::default() - }, - Err(message) => ScenarioOutcome { - error: Some(message), - ..ScenarioOutcome::default() - }, - } - } - }; - serde_json::to_string(&outcome).map_err(|error| error.to_string()) +) -> Result { + let ctes = match scan_top_level_ctes(sql, file, RelationshipSource::Scenario, syntax) { + TopLevelCtes::Failed(message) => return Err(message), + TopLevelCtes::Scanned(ctes) => ctes, + }; + if let Some(message) = independence_error( + &IndependenceCheck { + ctes: &ctes, + expected_prefix: EXPECTED_PREFIX, + assertion_prefix: ASSERT_PREFIX, + file_label: file, + context_label: SCENARIO_CONTEXT, + }, + syntax, + ) { + return Err(message); + } + classify(ctes, file) } -fn classify(ctes: Vec, file: &str) -> Result { - let mut scenario: ClassifiedScenario = ClassifiedScenario::default(); +fn classify(ctes: Vec, file: &str) -> Result { + let mut scenario: SqlScenarioCtes = SqlScenarioCtes::default(); for cte in ctes { let name: &str = &cte.0; if let Some(value) = name.strip_prefix(SOURCE_PREFIX) { diff --git a/crates/sqlbuild-analysis/src/compiler/main/mod.rs b/crates/sqlbuild-analysis/src/compiler/main/mod.rs index 8f17d7b413..0975b294d5 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/mod.rs @@ -1,7 +1,8 @@ pub mod function_type_validation; -pub mod sql_scenario_extraction; +pub mod sql_scenario_classification; pub mod sql_test_assembly; pub mod sql_test_chains; +pub mod sql_test_classification; pub mod sql_test_difference_sampling; pub mod sql_test_extraction; pub mod sql_test_fixture_facts; diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_classification.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_classification.rs new file mode 100644 index 0000000000..96857520d3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_classification.rs @@ -0,0 +1,15 @@ +//! A SQL scenario classified natively, as the compile attachment reads it. + +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::compiler::_helpers; +use crate::compiler::models::SqlScenarioCtes; + +/// The scenario's classified CTEs, or its error. +pub fn extract_scenario( + sql: &str, + file_label: &str, + syntax: &LexicalSyntax, +) -> Result { + _helpers::sql_tests::scenario_extraction::extract_scenario(sql, file_label, syntax) +} diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs deleted file mode 100644 index 9c2d99e7af..0000000000 --- a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs +++ /dev/null @@ -1,12 +0,0 @@ -use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; - -use crate::compiler::_helpers; - -/// The scenario's classification or error as JSON; `Err` holds the JSON encoder's message. -pub fn extract_scenario_json( - sql: &str, - file_label: &str, - syntax: &LexicalSyntax, -) -> Result { - _helpers::sql_tests::scenario_extraction::extract_scenario_json(sql, file_label, syntax) -} diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_test_classification.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_classification.rs new file mode 100644 index 0000000000..c6955e703e --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_classification.rs @@ -0,0 +1,21 @@ +//! SQL tests classified natively, as the compile attachment reads them. + +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::compiler::_helpers; +use crate::compiler::models::{SqlTestClassification, SqlTestExtractionFailure, SqlTestRequest}; + +/// Each test's classification, or the first test's failure, under the adapter's lexical rules. +pub fn extract_sql_tests( + tests: &[SqlTestRequest], + syntax: &LexicalSyntax, +) -> Result, SqlTestExtractionFailure> { + _helpers::sql_tests::extraction::extract_tests(tests, syntax).map_err(|(index, failure)| { + SqlTestExtractionFailure { + index, + message: failure.message, + help: failure.help, + token: failure.token, + } + }) +} diff --git a/crates/sqlbuild-analysis/src/compiler/models.rs b/crates/sqlbuild-analysis/src/compiler/models.rs index 457dd969a1..4902677c6c 100644 --- a/crates/sqlbuild-analysis/src/compiler/models.rs +++ b/crates/sqlbuild-analysis/src/compiler/models.rs @@ -308,3 +308,88 @@ pub enum SqlTestAssemblyFailure { /// An internal native failure. Internal(String), } + +/// One SQL test block to extract: its SQL, file label, mode and extraction stage. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestRequest { + pub sql: String, + pub file_label: String, + pub mode: String, + /// The test before macro and variable expansion, where macro calls are still authored. + #[serde(default)] + pub raw: bool, + /// Read only the CTE names and body offsets of the authored block, before any expansion. + #[serde(default)] + pub authored: bool, + /// The authored text is a scenario rather than a SQL test block. + #[serde(default)] + pub scenario: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SqlTestCteText(pub String, pub String); + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum SqlTestClassification { + /// Each CTE's name, its code-point offset in the authored block and its stripped body. + Authored { ctes: Vec<(String, usize, String)> }, + Model { + mode: String, + authored: Vec, + #[serde(rename = "macroMocks")] + macro_mocks: Vec<(String, String)>, + #[serde(rename = "mockModels")] + mock_models: Vec, + #[serde(rename = "mockSources")] + mock_sources: Vec, + #[serde(rename = "mockSeeds")] + mock_seeds: Vec, + #[serde(rename = "mockDbtRefs")] + mock_dbt_refs: Vec, + #[serde(rename = "mockTableFunctions")] + mock_table_functions: Vec, + expected: Vec, + #[serde(rename = "expectedModels")] + expected_models: Vec, + assertions: Vec, + #[serde(rename = "assertionNames")] + assertion_names: Vec, + }, + Direct { + mode: String, + helpers: Vec, + actual: SqlTestCteText, + expected: SqlTestCteText, + /// Whether a helper or the expected CTE has a malformed reference call (P012). + #[serde(rename = "invalidCalls")] + invalid_calls: bool, + }, +} + +/// Why extracting one test of a batch failed, and the authored text the author can act on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SqlTestExtractionFailure { + /// The failing test's position in the batch. + pub index: usize, + pub message: String, + pub help: Option, + /// The offending text and its code-point offset in the test SQL. + pub token: Option<(String, usize)>, +} + +/// One scenario's top-level CTEs, classified by role. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlScenarioCtes { + pub authored: Vec<(String, String)>, + pub expected: Vec<(String, String)>, + pub assertions: Vec<(String, String)>, + pub source_fixtures: Vec, + pub ref_fixtures: Vec, + pub seed_fixtures: Vec, + pub dbt_ref_fixtures: Vec, + pub expected_models: Vec, + pub assertion_names: Vec, +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index cc4183b024..602fb830b6 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -1,7 +1,5 @@ #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_sql_scenario_extraction.rs"] -mod sql_scenario_extraction; #[path = "tests/test_sql_test_cte_rename.rs"] mod sql_test_cte_rename; #[path = "tests/test_sql_test_cte_slices.rs"] diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index c348ab5536..b94a1c4d0f 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -133,12 +133,6 @@ pub(crate) struct HelperReferenceTestCase { pub(crate) expected_absent_fragments: &'static [&'static str], } -pub(crate) struct ScenarioExtractionTestCase { - pub(crate) description: &'static str, - pub(crate) sql: &'static str, - pub(crate) expected_json: &'static str, -} - pub(crate) struct UnresolvedReaderReferenceTestCase { pub(crate) description: &'static str, pub(crate) sql_analysis_enabled: bool, diff --git a/crates/sqlbuild-analysis/src/contracts/main/join_helps.rs b/crates/sqlbuild-analysis/src/contracts/main/join_helps.rs new file mode 100644 index 0000000000..a75f6daf69 --- /dev/null +++ b/crates/sqlbuild-analysis/src/contracts/main/join_helps.rs @@ -0,0 +1,9 @@ +//! Several help paragraphs joined into one help text. + +use crate::contracts::_helpers::setting_help; + +/// The paragraphs joined with one `= help:` label each. +#[must_use] +pub fn join_helps(helps: &[String]) -> String { + setting_help::join_helps(helps) +} diff --git a/crates/sqlbuild-analysis/src/contracts/main/mod.rs b/crates/sqlbuild-analysis/src/contracts/main/mod.rs index 320ccacdc3..2cea672288 100644 --- a/crates/sqlbuild-analysis/src/contracts/main/mod.rs +++ b/crates/sqlbuild-analysis/src/contracts/main/mod.rs @@ -1,2 +1,4 @@ pub mod evaluate_model_contracts; +pub mod join_helps; pub mod promotion_conflicts; +pub mod snippet_help; diff --git a/crates/sqlbuild-analysis/src/contracts/main/snippet_help.rs b/crates/sqlbuild-analysis/src/contracts/main/snippet_help.rs new file mode 100644 index 0000000000..5bb7fcf5c5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/contracts/main/snippet_help.rs @@ -0,0 +1,9 @@ +//! A diagnostic help that shows the exact lines to write. + +use crate::contracts::_helpers::setting_help; + +/// `, :` followed by the exact lines to write, one per line. +#[must_use] +pub fn snippet_help(purpose: &str, target: &str, lines: &[String]) -> String { + setting_help::snippet_help(purpose, target, lines) +} diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/close_matches.rs b/crates/sqlbuild-analysis/src/graph/_helpers/close_matches.rs new file mode 100644 index 0000000000..f8b81000db --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/_helpers/close_matches.rs @@ -0,0 +1,118 @@ +//! Python's `difflib.get_close_matches` over characters, for selector name suggestions. + +use std::collections::HashMap; + +use crate::graph::constants::{AUTOJUNK_DIVISOR, AUTOJUNK_MIN_LENGTH}; + +/// `difflib.SequenceMatcher(None, a, word)` state for one fixed `word`. +struct Matcher<'word> { + word: &'word [char], + positions: HashMap>, +} + +impl<'word> Matcher<'word> { + fn new(word: &'word [char]) -> Self { + let mut positions: HashMap> = HashMap::new(); + for (index, character) in word.iter().enumerate() { + positions.entry(*character).or_default().push(index); + } + if word.len() >= AUTOJUNK_MIN_LENGTH { + let limit: usize = word.len() / AUTOJUNK_DIVISOR + 1; + positions.retain(|_, indexes| indexes.len() <= limit); + } + Self { word, positions } + } + + /// `find_longest_match` with no junk: `(start in a, start in word, size)`. + fn longest(&self, a: &[char], bounds: (usize, usize, usize, usize)) -> (usize, usize, usize) { + let (a_low, a_high, b_low, b_high) = bounds; + let (mut best_a, mut best_b, mut best_size) = (a_low, b_low, 0); + let mut lengths: HashMap = HashMap::new(); + for (offset, character) in a[a_low..a_high].iter().enumerate() { + let index: usize = a_low + offset; + let mut next: HashMap = HashMap::new(); + for &position in self.positions.get(character).map_or(&[][..], Vec::as_slice) { + if position < b_low { + continue; + } + if position >= b_high { + break; + } + let previous: usize = position + .checked_sub(1) + .and_then(|before| lengths.get(&before).copied()) + .unwrap_or(0); + let size: usize = previous + 1; + next.insert(position, size); + if size > best_size { + (best_a, best_b, best_size) = (index + 1 - size, position + 1 - size, size); + } + } + lengths = next; + } + while best_a > a_low && best_b > b_low && a[best_a - 1] == self.word[best_b - 1] { + (best_a, best_b, best_size) = (best_a - 1, best_b - 1, best_size + 1); + } + while best_a + best_size < a_high + && best_b + best_size < b_high + && a[best_a + best_size] == self.word[best_b + best_size] + { + best_size += 1; + } + (best_a, best_b, best_size) + } + + /// The total size of `get_matching_blocks`. + fn matched(&self, a: &[char]) -> usize { + let mut total: usize = 0; + let mut pending: Vec<(usize, usize, usize, usize)> = vec![(0, a.len(), 0, self.word.len())]; + while let Some(bounds) = pending.pop() { + let (a_low, a_high, b_low, b_high) = bounds; + let (start_a, start_b, size) = self.longest(a, bounds); + if size == 0 { + continue; + } + total += size; + if a_low < start_a && b_low < start_b { + pending.push((a_low, start_a, b_low, start_b)); + } + if start_a + size < a_high && start_b + size < b_high { + pending.push((start_a + size, a_high, start_b + size, b_high)); + } + } + total + } + + fn ratio(&self, a: &[char]) -> f64 { + let length: usize = a.len() + self.word.len(); + if length == 0 { + return 1.0; + } + 2.0 * self.matched(a) as f64 / length as f64 + } +} + +/// The best `limit` candidates scoring at least `cutoff`, best first, as Python orders them. +pub(crate) fn close_matches( + word: &str, + candidates: &[&str], + limit: usize, + cutoff: f64, +) -> Vec { + let word: Vec = word.chars().collect(); + let matcher: Matcher<'_> = Matcher::new(&word); + let mut scored: Vec<(f64, &str)> = candidates + .iter() + .map(|candidate| { + let characters: Vec = candidate.chars().collect(); + (matcher.ratio(&characters), *candidate) + }) + .filter(|(score, _)| *score >= cutoff) + .collect(); + scored.sort_by(|left, right| right.0.total_cmp(&left.0).then_with(|| right.1.cmp(left.1))); + scored + .into_iter() + .take(limit) + .map(|(_, candidate)| candidate.to_owned()) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/closure.rs b/crates/sqlbuild-analysis/src/graph/_helpers/closure.rs new file mode 100644 index 0000000000..1c9360819b --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/_helpers/closure.rs @@ -0,0 +1,70 @@ +//! Python's breadth-first transitive closure and directed path nodes over graph edges. + +use std::collections::{BTreeSet, HashSet, VecDeque}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::models::{EdgeDirection as Direction, ProjectGraph}; + +impl ProjectGraph { + pub(crate) fn edges(&self, key: &ObjectKey, direction: Direction) -> &[ObjectKey] { + let (entries, index) = match direction { + Direction::Upstream => (&self.upstream, &self.upstream_index), + Direction::Downstream => (&self.downstream, &self.downstream_index), + }; + index + .get(key) + .map_or(&[], |&position| entries[position].1.as_slice()) + } + + /// Every key reachable from `start`; `start` itself only through a cycle. + pub(crate) fn closure(&self, start: &ObjectKey, direction: Direction) -> BTreeSet { + let mut visited: HashSet<&ObjectKey> = HashSet::new(); + let mut frontier: VecDeque<&ObjectKey> = VecDeque::from([start]); + while let Some(current) = frontier.pop_front() { + for neighbor in self.edges(current, direction) { + if visited.insert(neighbor) { + frontier.push_back(neighbor); + } + } + } + visited.into_iter().cloned().collect() + } + + /// Keys on directed paths from `start` to `end`, or `None` when `end` is not downstream. + pub(crate) fn path_nodes( + &self, + start: &ObjectKey, + end: &ObjectKey, + ) -> Option> { + let reachable: BTreeSet = self.closure(start, Direction::Downstream); + if !reachable.contains(end) { + return None; + } + let mut upstream_of_end: BTreeSet = BTreeSet::new(); + let mut stack: Vec = vec![end.clone()]; + while let Some(current) = stack.pop() { + if !upstream_of_end.insert(current.clone()) { + continue; + } + for parent in self.downstream_parents(¤t) { + if reachable.contains(&parent) || &parent == start { + stack.push(parent); + } + } + } + let mut nodes: BTreeSet = + reachable.intersection(&upstream_of_end).cloned().collect(); + nodes.insert(start.clone()); + nodes.insert(end.clone()); + Some(nodes) + } + + /// Keys whose downstream list holds `key`: the inverse of the downstream edges. + fn downstream_parents(&self, key: &ObjectKey) -> Vec { + self.downstream + .iter() + .filter(|(_, children)| children.contains(key)) + .map(|(parent, _)| parent.clone()) + .collect() + } +} diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/indexes.rs b/crates/sqlbuild-analysis/src/graph/_helpers/indexes.rs new file mode 100644 index 0000000000..b5689372f9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/_helpers/indexes.rs @@ -0,0 +1,163 @@ +//! Python's lineage edges, tag, path and name indexes over one project's resources. + +use std::collections::{HashMap, HashSet}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::constants::{MODEL_ROOT, MODEL_ROOT_PREFIX, SQL_TEST_RESOURCE}; +use crate::graph::models::{GraphIndexes, GraphResource, ProjectGraph}; + +type Edges = Vec<(ObjectKey, Vec)>; + +/// Insertion-ordered key lists, as Python's `dict.setdefault(key, []).append(...)` builds them. +#[derive(Default)] +struct OrderedEdges { + entries: Edges, + index: HashMap, +} + +impl OrderedEdges { + /// The list stored for `key`, appended empty in insertion order when the key is new. + fn list(&mut self, key: &ObjectKey) -> &mut Vec { + let position: usize = match self.index.get(key) { + Some(&position) => position, + None => { + self.entries.push((key.clone(), Vec::new())); + self.index.insert(key.clone(), self.entries.len() - 1); + self.entries.len() - 1 + } + }; + &mut self.entries[position].1 + } +} + +/// Tag to keys in first-tagged order, each key once, as Python's tag sets hold them. +#[derive(Default)] +struct OrderedTags { + entries: Vec<(String, Vec)>, + index: HashMap)>, +} + +impl OrderedTags { + fn add(&mut self, tag: &str, key: &ObjectKey) { + let (position, seen) = self.index.entry(tag.to_owned()).or_insert_with(|| { + self.entries.push((tag.to_owned(), Vec::new())); + (self.entries.len() - 1, HashSet::new()) + }); + if seen.insert(key.clone()) { + self.entries[*position].1.push(key.clone()); + } + } +} + +impl ProjectGraph { + /// Point `key`'s name at it; a later resource with the same name replaces the key in place. + fn set_name(&mut self, name: &str, key: &ObjectKey) { + match self.name_index.get(name) { + Some(&position) => self.names[position].1 = key.clone(), + None => { + self.name_index.insert(name.to_owned(), self.names.len()); + self.names.push((name.to_owned(), key.clone())); + } + } + } +} + +pub(crate) fn build(resources: &[GraphResource]) -> ProjectGraph { + let mut upstream: OrderedEdges = OrderedEdges::default(); + for resource in resources + .iter() + .filter(|resource| resource.key.0 != SQL_TEST_RESOURCE) + { + *upstream.list(&resource.key) = lineage_deps(&resource.deps); + } + let downstream: Edges = downstream(&upstream.entries); + let mut tags: OrderedTags = OrderedTags::default(); + let mut graph = ProjectGraph { + downstream_index: positions(&downstream), + downstream, + upstream_index: upstream.index, + upstream: upstream.entries, + ..ProjectGraph::default() + }; + for resource in resources { + for tag in &resource.tags { + tags.add(tag, &resource.key); + } + if let Some(folder) = &resource.folder { + graph + .paths + .push((resource.key.clone(), below_model_root(folder))); + } + graph.set_name(&resource.key.1, &resource.key); + } + graph.tags = tags.entries; + graph +} + +/// A graph over indexes a caller already built, such as the planner's. +pub(crate) fn from_indexes(indexes: GraphIndexes) -> ProjectGraph { + let GraphIndexes { + names, + upstream, + downstream, + tags, + paths, + } = indexes; + let mut graph = ProjectGraph { + upstream_index: positions(&upstream), + downstream_index: positions(&downstream), + upstream, + downstream, + tags, + paths, + ..ProjectGraph::default() + }; + for (name, key) in names { + graph.set_name(&name, &key); + } + graph +} + +fn lineage_deps(deps: &[ObjectKey]) -> Vec { + deps.iter() + .filter(|dep| dep.0 != SQL_TEST_RESOURCE) + .cloned() + .collect() +} + +/// Inverted edges keyed in upstream order, each list sorted by `(resource type, name)`. +fn downstream(upstream: &[(ObjectKey, Vec)]) -> Edges { + let mut inverted: OrderedEdges = OrderedEdges::default(); + for (key, _) in upstream { + let _ = inverted.list(key); + } + for (key, deps) in upstream { + for dep in deps { + inverted.list(dep).push(key.clone()); + } + } + let mut entries: Edges = inverted.entries; + for (_, keys) in &mut entries { + keys.sort(); + } + entries +} + +fn positions(entries: &[(ObjectKey, Vec)]) -> HashMap { + entries + .iter() + .enumerate() + .map(|(position, (key, _))| (key.clone(), position)) + .collect() +} + +/// A model directory with forward slashes and the leading `models/` removed. +fn below_model_root(folder: &str) -> String { + let folder: String = folder.replace('\\', "/"); + if folder == MODEL_ROOT { + return String::new(); + } + folder + .strip_prefix(MODEL_ROOT_PREFIX) + .map_or_else(|| folder.clone(), str::to_owned) +} diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/layers.rs b/crates/sqlbuild-analysis/src/graph/_helpers/layers.rs new file mode 100644 index 0000000000..452c7666e0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/_helpers/layers.rs @@ -0,0 +1,73 @@ +//! Python's compile-report execution layer count over the model subgraph. + +use std::collections::{HashMap, HashSet}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::constants::MODEL_RESOURCE; +use crate::graph::models::{EdgeDirection, ProjectGraph}; + +/// Layers of a breadth-first topological walk over models; a cycle still counts at least one. +pub(crate) fn model_layer_count(graph: &ProjectGraph) -> usize { + let models: HashSet<&ObjectKey> = graph + .upstream + .iter() + .map(|(key, _)| key) + .filter(|key| key.0 == MODEL_RESOURCE) + .collect(); + if models.is_empty() { + return 0; + } + let mut remaining: HashMap<&ObjectKey, HashSet<&ObjectKey>> = models + .iter() + .map(|&key| { + ( + key, + model_edges(graph, &models, key, EdgeDirection::Upstream), + ) + }) + .collect(); + let mut layer: HashSet<&ObjectKey> = remaining + .iter() + .filter(|(_, deps)| deps.is_empty()) + .map(|(&key, _)| key) + .collect(); + let mut visited: HashSet<&ObjectKey> = HashSet::new(); + let mut layers: usize = 0; + while !layer.is_empty() { + layers += 1; + let mut next: HashSet<&ObjectKey> = HashSet::new(); + for key in layer { + visited.insert(key); + for downstream in model_edges(graph, &models, key, EdgeDirection::Downstream) { + if visited.contains(downstream) { + continue; + } + if let Some(deps) = remaining.get_mut(downstream) { + deps.remove(key); + if deps.is_empty() { + next.insert(downstream); + } + } + } + } + layer = next; + } + if visited.len() == models.len() { + layers + } else { + layers.max(1) + } +} + +fn model_edges<'graph>( + graph: &'graph ProjectGraph, + models: &HashSet<&ObjectKey>, + key: &ObjectKey, + direction: EdgeDirection, +) -> HashSet<&'graph ObjectKey> { + graph + .edges(key, direction) + .iter() + .filter(|dependency| models.contains(dependency)) + .collect() +} diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/mod.rs b/crates/sqlbuild-analysis/src/graph/_helpers/mod.rs new file mode 100644 index 0000000000..18a62b56e5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod close_matches; +pub(crate) mod closure; +pub(crate) mod indexes; +pub(crate) mod layers; +pub(crate) mod selectors; +pub(crate) mod unique_ids; diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs b/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs new file mode 100644 index 0000000000..e6aa2a4c89 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs @@ -0,0 +1,388 @@ +//! Python's planner selector grammar and resolution over the project graph. + +use std::collections::BTreeSet; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::_helpers::close_matches::close_matches; +use crate::graph::constants::{ + EMPTY_SELECTOR_CODE, EMPTY_VALUE_CODE, EXPANSION_MARKER, FOLDER_SEPARATOR, + INTERSECTION_SEPARATOR, KIND_SEPARATOR, MISPLACED_MARKER_CODE, MISSING_NAME_CODE, + MODEL_RESOURCE, MODEL_ROOT, MODEL_ROOT_PREFIX, NAME_KIND, PATH_KIND, PATH_ROOT_CODE, + PATH_ROOT_ERROR, PATH_SELECTOR_CODE, PATH_SEPARATOR, PATTERN_CHARACTERS, PLANNER_DEFAULT_CODE, + SEED_RESOURCE, SOURCE_RESOURCE, SQL_FILE_SUFFIX, SUGGESTION_CUTOFF, SUGGESTION_LIMIT, + TABLE_FUNCTION_RESOURCE, TAG_KIND, TEST_KIND, UDF_RESOURCE, UNIT_TEST_CODE, + UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_KIND_CODE, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, + UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, UNMAPPED_KINDS, +}; +use crate::graph::errors::SelectorError; +use crate::graph::models::{ + BuildResources, EdgeDirection as Direction, ParsedSelector, ProjectGraph, +}; +use sqlbuild_core::text::main::python_fnmatchcase::python_fnmatchcase; + +pub(crate) type Keys = BTreeSet; +type Resolved = Result; + +fn error(code: &'static str, message: String) -> SelectorError { + SelectorError { + code, + message, + help: None, + } +} + +pub(crate) fn resolve( + graph: &ProjectGraph, + select: &[String], + exclude: &[String], +) -> Result, SelectorError> { + let all: Keys = graph.names.iter().map(|(_, key)| key.clone()).collect(); + if select.is_empty() && exclude.is_empty() { + return Ok(all.into_iter().collect()); + } + let selected: Keys = if select.is_empty() { + all + } else { + tokens(graph, select)? + }; + let excluded: Keys = tokens(graph, exclude)?; + let scoped: Keys = selected.difference(&excluded).cloned().collect(); + Ok(build_resources(graph, &scoped, BuildResources::SELECTION) + .into_iter() + .collect()) +} + +/// Python's `expand_required_build_resources`: functions and seeds a selected scope needs. +pub(crate) fn build_resources( + graph: &ProjectGraph, + selected: &Keys, + include: BuildResources, +) -> Keys { + let mut expanded: Keys = selected.clone(); + for key in selected { + for upstream in graph.closure(key, Direction::Upstream) { + let wanted: bool = (include.upstream_functions && is_function(&upstream)) + || (include.upstream_seeds && upstream.0 == SEED_RESOURCE); + if wanted { + expanded.insert(upstream); + } + } + } + if include.downstream_functions { + for key in selected.iter().filter(|key| key.0 == MODEL_RESOURCE) { + expanded.extend( + graph + .edges(key, Direction::Downstream) + .iter() + .filter(|key| is_function(key)) + .cloned(), + ); + } + } + expanded +} + +fn is_function(key: &ObjectKey) -> bool { + key.0 == UDF_RESOURCE || key.0 == TABLE_FUNCTION_RESOURCE +} + +/// The union of whitespace-separated tokens across `selectors`, without build expansion. +pub(crate) fn tokens(graph: &ProjectGraph, selectors: &[String]) -> Resolved { + let mut resolved: Keys = Keys::new(); + for selector in selectors { + for token in selector + .split(python_space) + .filter(|token| !token.is_empty()) + { + resolved.extend(intersection(graph, token)?); + } + } + Ok(resolved) +} + +fn intersection(graph: &ProjectGraph, token: &str) -> Resolved { + let mut parts = token.split(INTERSECTION_SEPARATOR); + let mut result: Keys = single(graph, parts.next().unwrap_or_default())?; + for part in parts { + let next: Keys = single(graph, part)?; + result = result.intersection(&next).cloned().collect(); + } + Ok(result) +} + +fn single(graph: &ProjectGraph, raw: &str) -> Resolved { + match parse(raw)? { + ParsedSelector::Path { + start, + end, + upstream, + downstream, + } => path(graph, (&start, &end), (upstream, downstream)), + ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + } => { + let keys: Keys = matched(graph, &kind, &value)?; + let mut result: Keys = keys.clone(); + for key in &keys { + if upstream { + result.extend(graph.closure(key, Direction::Upstream)); + } + if downstream { + result.extend(graph.closure(key, Direction::Downstream)); + } + } + Ok(result) + } + } +} + +fn path(graph: &ProjectGraph, names: (&str, &str), expand: (bool, bool)) -> Resolved { + let start: ObjectKey = named(graph, names.0)?; + let end: ObjectKey = named(graph, names.1)?; + let Some(mut result) = graph.path_nodes(&start, &end) else { + return Err(error( + PLANNER_DEFAULT_CODE, + format!( + "'{}:{}' is not downstream of '{}:{}'", + end.0, end.1, start.0, start.1 + ), + )); + }; + if expand.0 { + result.extend(graph.closure(&start, Direction::Upstream)); + } + if expand.1 { + result.extend(graph.closure(&end, Direction::Downstream)); + } + Ok(result) +} + +fn named(graph: &ProjectGraph, name: &str) -> Result { + graph + .key_named(name) + .cloned() + .ok_or_else(|| error(UNKNOWN_NAME_CODE, format!("unknown selector name '{name}'"))) +} + +pub(crate) fn parse(raw: &str) -> Result { + let stripped: &str = raw.trim_matches(python_space); + if stripped.is_empty() { + return Err(error(EMPTY_SELECTOR_CODE, "empty selector".to_owned())); + } + let upstream: bool = stripped.starts_with(EXPANSION_MARKER); + let downstream: bool = stripped.ends_with(EXPANSION_MARKER); + let core: &str = stripped + .trim_start_matches(EXPANSION_MARKER) + .trim_end_matches(EXPANSION_MARKER); + if core.is_empty() { + return Err(error( + MISSING_NAME_CODE, + format!("selector '{stripped}' has no name after removing '+' markers"), + )); + } + if core.contains(EXPANSION_MARKER) { + return Err(error( + MISPLACED_MARKER_CODE, + format!("selector '{stripped}' contains '+' in an unsupported position"), + )); + } + if let Some((start, end)) = core.split_once(PATH_SEPARATOR) { + let (start, end) = ( + start.trim_matches(python_space), + end.trim_matches(python_space), + ); + if start.is_empty() || end.is_empty() { + return Err(error( + PATH_SELECTOR_CODE, + format!("path selector '{stripped}' requires names on both sides of '~'"), + )); + } + return Ok(ParsedSelector::Path { + start: start.to_owned(), + end: end.to_owned(), + upstream, + downstream, + }); + } + let (kind, value): (&str, &str) = match core.split_once(KIND_SEPARATOR) { + Some((prefix, value)) => { + if !is_selector_kind(prefix) { + return Err(error( + UNKNOWN_KIND_CODE, + format!("unknown selector type '{prefix}' in '{stripped}'"), + )); + } + if value.is_empty() { + return Err(error( + EMPTY_VALUE_CODE, + format!("selector '{stripped}' has empty value after ':'"), + )); + } + (prefix, value) + } + None if core.contains(FOLDER_SEPARATOR) => (PATH_KIND, core.trim_matches(FOLDER_SEPARATOR)), + None => (NAME_KIND, core), + }; + Ok(ParsedSelector::Kind { + kind: kind.to_owned(), + value: value.to_owned(), + upstream, + downstream, + }) +} + +fn is_selector_kind(prefix: &str) -> bool { + [ + SEED_RESOURCE, + SOURCE_RESOURCE, + TAG_KIND, + PATH_KIND, + TEST_KIND, + ] + .contains(&prefix) + || UNMAPPED_KINDS.contains(&prefix) +} + +/// Python's `match_selector_keys`: the keys one parsed selector names before `+` expansion. +pub(crate) fn matched(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved { + match kind { + TAG_KIND => { + let keys: Keys = graph.tagged(value); + if keys.is_empty() { + return Err(error( + UNKNOWN_TAG_CODE, + format!("no models found with tag '{value}'"), + )); + } + Ok(keys) + } + PATH_KIND => folder(graph, value), + TEST_KIND => Err(error( + UNIT_TEST_CODE, + format!( + "selector '{TEST_KIND}{KIND_SEPARATOR}{value}' selects a unit test; \ + {UNIT_TEST_ONLY_TEST_AND_BUILD}" + ), + )), + _ => { + let keys: Keys = lookup(graph, kind, value)?; + if !keys.is_empty() { + return Ok(keys); + } + let pattern: bool = is_pattern(value); + let names: Vec<&str> = graph.names.iter().map(|(name, _)| name.as_str()).collect(); + Err(SelectorError { + code: UNKNOWN_NAME_CODE, + message: format!( + "unknown selector {} '{value}'", + if pattern { "pattern" } else { "name" } + ), + help: (!pattern).then(|| suggestion(value, names)).flatten(), + }) + } + } +} + +pub(crate) fn suggestion(value: &str, mut names: Vec<&str>) -> Option { + names.sort_unstable(); + let matches: Vec = close_matches(value, &names, SUGGESTION_LIMIT, SUGGESTION_CUTOFF); + if matches.is_empty() { + return None; + } + let quoted: Vec = matches.iter().map(|name| format!("'{name}'")).collect(); + Some(format!("did you mean {}?", quoted.join(", "))) +} + +fn lookup(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved { + let resource: Option<&str> = match kind { + NAME_KIND => None, + SEED_RESOURCE | SOURCE_RESOURCE => Some(kind), + _ => { + return Err(error( + UNMAPPED_KIND_CODE, + format!("selector type '{kind}' does not map to a resource type yet"), + )); + } + }; + let kind_matches = |key: &ObjectKey| resource.is_none_or(|resource| key.0 == resource); + if !is_pattern(value) { + return Ok(graph + .key_named(value) + .filter(|key| kind_matches(key)) + .cloned() + .into_iter() + .collect()); + } + Ok(graph + .names + .iter() + .filter(|(name, key)| kind_matches(key) && glob_matches(value, name)) + .map(|(_, key)| key.clone()) + .collect()) +} + +fn folder(graph: &ProjectGraph, value: &str) -> Resolved { + let folder: String = value.replace('\\', "/").trim_matches('/').to_owned(); + let selector: &str = if folder == MODEL_ROOT { + "" + } else if let Some(rest) = folder.strip_prefix(MODEL_ROOT_PREFIX) { + rest + } else { + return Err(error(PATH_ROOT_CODE, PATH_ROOT_ERROR.to_owned())); + }; + let keys: Keys = graph + .paths + .iter() + .filter(|(_, indexed)| { + selector.is_empty() + || indexed == selector + || indexed.starts_with(&format!("{selector}/")) + }) + .map(|(key, _)| key.clone()) + .collect(); + if keys.is_empty() { + let hint: &str = if folder.ends_with(SQL_FILE_SUFFIX) { + " Path selectors match folders; select a single model by its name." + } else { + "" + }; + return Err(error( + UNKNOWN_PATH_CODE, + format!("no models found under path '{folder}'.{hint}"), + )); + } + Ok(keys) +} + +/// Python's `str.isspace`, which also counts the information separators U+001C..U+001F. +fn python_space(character: char) -> bool { + character.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&character) +} + +pub(crate) fn is_pattern(value: &str) -> bool { + value.contains(PATTERN_CHARACTERS) +} + +impl ProjectGraph { + pub(crate) fn key_named(&self, name: &str) -> Option<&ObjectKey> { + self.name_index + .get(name) + .map(|&position| &self.names[position].1) + } + + fn tagged(&self, tag: &str) -> Keys { + self.tags + .iter() + .find(|(name, _)| name == tag) + .map(|(_, keys)| keys.iter().cloned().collect()) + .unwrap_or_default() + } +} + +/// Python's `fnmatch.fnmatchcase`: `*`, `?`, `[seq]` and `[!seq]` over characters. +fn glob_matches(pattern: &str, name: &str) -> bool { + python_fnmatchcase(name, pattern) +} diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/unique_ids.rs b/crates/sqlbuild-analysis/src/graph/_helpers/unique_ids.rs new file mode 100644 index 0000000000..75c308f7ca --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/_helpers/unique_ids.rs @@ -0,0 +1,45 @@ +//! Edges written as dbt-style `prefix.project.name` unique ids, as the manifest stores them. + +use std::collections::HashMap; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::constants::DEFAULT_UNIQUE_ID_PREFIX; +use crate::graph::models::{EdgeDirection, ProjectGraph}; + +pub(crate) fn unique_id_edges( + graph: &ProjectGraph, + direction: EdgeDirection, + project_name: &str, + prefixes: &HashMap, +) -> Vec<(String, Vec)> { + let entries: &[(ObjectKey, Vec)] = match direction { + EdgeDirection::Upstream => &graph.upstream, + EdgeDirection::Downstream => &graph.downstream, + }; + entries + .iter() + .map(|(key, neighbours)| { + ( + unique_id(key, project_name, prefixes), + unique_ids(neighbours, project_name, prefixes), + ) + }) + .collect() +} + +fn unique_ids( + keys: &[ObjectKey], + project_name: &str, + prefixes: &HashMap, +) -> Vec { + keys.iter() + .map(|key| unique_id(key, project_name, prefixes)) + .collect() +} + +fn unique_id(key: &ObjectKey, project_name: &str, prefixes: &HashMap) -> String { + let prefix: &str = prefixes + .get(&key.0) + .map_or(DEFAULT_UNIQUE_ID_PREFIX, String::as_str); + format!("{prefix}.{project_name}.{}", key.1) +} diff --git a/crates/sqlbuild-analysis/src/graph/constants.rs b/crates/sqlbuild-analysis/src/graph/constants.rs new file mode 100644 index 0000000000..f7a27d16c1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/constants.rs @@ -0,0 +1,46 @@ +//! Python's planner selector syntax, error codes and messages the graph reproduces. + +pub(crate) const MODEL_RESOURCE: &str = "model"; +pub(crate) const SOURCE_RESOURCE: &str = "source"; +pub(crate) const SEED_RESOURCE: &str = "seed"; +pub(crate) const UDF_RESOURCE: &str = "udf"; +pub(crate) const TABLE_FUNCTION_RESOURCE: &str = "table_fn"; +pub(crate) const SQL_TEST_RESOURCE: &str = "sql_test"; +pub(crate) const EXPANSION_MARKER: char = '+'; +pub(crate) const PATH_SEPARATOR: char = '~'; +pub(crate) const KIND_SEPARATOR: char = ':'; +pub(crate) const FOLDER_SEPARATOR: char = '/'; +pub(crate) const INTERSECTION_SEPARATOR: char = ','; +pub(crate) const PATTERN_CHARACTERS: [char; 3] = ['*', '?', '[']; +pub(crate) const MODEL_ROOT: &str = "models"; +pub(crate) const MODEL_ROOT_PREFIX: &str = "models/"; +pub(crate) const SQL_FILE_SUFFIX: &str = ".sql"; +pub(crate) const TAG_KIND: &str = "tag"; +pub(crate) const PATH_KIND: &str = "path"; +pub(crate) const TEST_KIND: &str = "test"; +/// Selector kinds Python parses but `sqb compile` cannot map to a compiled resource. +pub(crate) const UNMAPPED_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +pub(crate) const SUGGESTION_CUTOFF: f64 = 0.8; +pub(crate) const SUGGESTION_LIMIT: usize = 3; +pub(crate) const EMPTY_SELECTOR_CODE: &str = "S001"; +pub(crate) const MISPLACED_MARKER_CODE: &str = "S002"; +pub(crate) const PATH_SELECTOR_CODE: &str = "S003"; +pub(crate) const MISSING_NAME_CODE: &str = "S004"; +pub(crate) const UNKNOWN_KIND_CODE: &str = "S005"; +pub(crate) const EMPTY_VALUE_CODE: &str = "S006"; +pub(crate) const UNKNOWN_NAME_CODE: &str = "S007"; +pub(crate) const UNKNOWN_TAG_CODE: &str = "S008"; +pub(crate) const UNKNOWN_PATH_CODE: &str = "S009"; +pub(crate) const UNMAPPED_KIND_CODE: &str = "S010"; +pub(crate) const PATH_ROOT_CODE: &str = "S012"; +pub(crate) const UNIT_TEST_CODE: &str = "S013"; +pub(crate) const PLANNER_DEFAULT_CODE: &str = "S000"; +pub(crate) const PATH_ROOT_ERROR: &str = + "path selectors require an explicit root: use 'models/' or 'python/'"; +pub(crate) const UNIT_TEST_ONLY_TEST_AND_BUILD: &str = + "only `sqb test` and `sqb build` accept unit-test selectors"; +/// `difflib.SequenceMatcher` treats an element as popular only in sequences this long. +pub(crate) const AUTOJUNK_MIN_LENGTH: usize = 200; +pub(crate) const AUTOJUNK_DIVISOR: usize = 100; +pub(crate) const NAME_KIND: &str = "name"; +pub(crate) const DEFAULT_UNIQUE_ID_PREFIX: &str = "model"; diff --git a/crates/sqlbuild-analysis/src/graph/errors.rs b/crates/sqlbuild-analysis/src/graph/errors.rs new file mode 100644 index 0000000000..d1f83580a8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/errors.rs @@ -0,0 +1,9 @@ +//! Selector failures. + +/// Python's `PlannerInputError` for a selector: code, message and optional help. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SelectorError { + pub code: &'static str, + pub message: String, + pub help: Option, +} diff --git a/crates/sqlbuild-analysis/src/graph/main/build_project_graph.rs b/crates/sqlbuild-analysis/src/graph/main/build_project_graph.rs new file mode 100644 index 0000000000..0be2c14045 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/build_project_graph.rs @@ -0,0 +1,6 @@ +use crate::graph::models::{GraphResource, ProjectGraph}; + +/// The graph indexes Python builds from a compiled project's models, sources, seeds and functions. +pub fn build_project_graph(resources: &[GraphResource]) -> ProjectGraph { + crate::graph::_helpers::indexes::build(resources) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/build_resources.rs b/crates/sqlbuild-analysis/src/graph/main/build_resources.rs new file mode 100644 index 0000000000..68668e59c4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/build_resources.rs @@ -0,0 +1,15 @@ +use crate::assembly::project::types::ObjectKey; +use crate::graph::_helpers::selectors::Keys; +use crate::graph::models::{BuildResources, ProjectGraph}; + +/// The selected keys plus the functions and seeds Python adds to build them, sorted. +pub fn build_resources( + graph: &ProjectGraph, + selected: &[ObjectKey], + include: BuildResources, +) -> Vec { + let selected: Keys = selected.iter().cloned().collect(); + crate::graph::_helpers::selectors::build_resources(graph, &selected, include) + .into_iter() + .collect() +} diff --git a/crates/sqlbuild-analysis/src/graph/main/edge_keys.rs b/crates/sqlbuild-analysis/src/graph/main/edge_keys.rs new file mode 100644 index 0000000000..af0c3c190b --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/edge_keys.rs @@ -0,0 +1,11 @@ +use crate::assembly::project::types::ObjectKey; +use crate::graph::models::{EdgeDirection, ProjectGraph}; + +/// A key's direct lineage neighbours in graph order; empty for a key the graph does not hold. +pub fn edge_keys<'graph>( + graph: &'graph ProjectGraph, + key: &ObjectKey, + direction: EdgeDirection, +) -> &'graph [ObjectKey] { + graph.edges(key, direction) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/graph_from_indexes.rs b/crates/sqlbuild-analysis/src/graph/main/graph_from_indexes.rs new file mode 100644 index 0000000000..7075445dcd --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/graph_from_indexes.rs @@ -0,0 +1,6 @@ +use crate::graph::models::{GraphIndexes, ProjectGraph}; + +/// A graph over indexes the caller built, keeping their order. +pub fn graph_from_indexes(indexes: GraphIndexes) -> ProjectGraph { + crate::graph::_helpers::indexes::from_indexes(indexes) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/match_selector.rs b/crates/sqlbuild-analysis/src/graph/main/match_selector.rs new file mode 100644 index 0000000000..60e9e263aa --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/match_selector.rs @@ -0,0 +1,13 @@ +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::ProjectGraph; + +/// Python's `match_selector_keys`: the keys a `kind:value` selector names before `+` expansion. +pub fn match_selector( + graph: &ProjectGraph, + kind: &str, + value: &str, +) -> Result, SelectorError> { + crate::graph::_helpers::selectors::matched(graph, kind, value) + .map(|keys| keys.into_iter().collect()) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/mod.rs b/crates/sqlbuild-analysis/src/graph/main/mod.rs new file mode 100644 index 0000000000..f461ab6399 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/mod.rs @@ -0,0 +1,11 @@ +pub mod build_project_graph; +pub mod build_resources; +pub mod edge_keys; +pub mod graph_from_indexes; +pub mod match_selector; +pub mod model_layer_count; +pub mod parse_selector; +pub mod resolve_selector_tokens; +pub mod resolve_selectors; +pub mod selector_name_help; +pub mod unique_id_edges; diff --git a/crates/sqlbuild-analysis/src/graph/main/model_layer_count.rs b/crates/sqlbuild-analysis/src/graph/main/model_layer_count.rs new file mode 100644 index 0000000000..4ab949a5c0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/model_layer_count.rs @@ -0,0 +1,6 @@ +use crate::graph::models::ProjectGraph; + +/// How many execution layers the compile report counts over the project's models. +pub fn model_layer_count(graph: &ProjectGraph) -> usize { + crate::graph::_helpers::layers::model_layer_count(graph) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/parse_selector.rs b/crates/sqlbuild-analysis/src/graph/main/parse_selector.rs new file mode 100644 index 0000000000..96dd46fb22 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/parse_selector.rs @@ -0,0 +1,7 @@ +use crate::graph::errors::SelectorError; +use crate::graph::models::ParsedSelector; + +/// Python's `parse_selector` for one token. +pub fn parse_selector(raw: &str) -> Result { + crate::graph::_helpers::selectors::parse(raw) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/resolve_selector_tokens.rs b/crates/sqlbuild-analysis/src/graph/main/resolve_selector_tokens.rs new file mode 100644 index 0000000000..02b205ad81 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/resolve_selector_tokens.rs @@ -0,0 +1,12 @@ +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::ProjectGraph; + +/// The keys whitespace-separated selector tokens match, without build resources, sorted. +pub fn resolve_selector_tokens( + graph: &ProjectGraph, + selectors: &[String], +) -> Result, SelectorError> { + crate::graph::_helpers::selectors::tokens(graph, selectors) + .map(|keys| keys.into_iter().collect()) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/resolve_selectors.rs b/crates/sqlbuild-analysis/src/graph/main/resolve_selectors.rs new file mode 100644 index 0000000000..f74dcecd1e --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/resolve_selectors.rs @@ -0,0 +1,12 @@ +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::ProjectGraph; + +/// The keys `--select` minus `--exclude` resolve to, with their build functions, sorted. +pub fn resolve_selectors( + graph: &ProjectGraph, + select: &[String], + exclude: &[String], +) -> Result, SelectorError> { + crate::graph::_helpers::selectors::resolve(graph, select, exclude) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/selector_name_help.rs b/crates/sqlbuild-analysis/src/graph/main/selector_name_help.rs new file mode 100644 index 0000000000..f5c42e7567 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/selector_name_help.rs @@ -0,0 +1,7 @@ +/// Python's `selector_name_help`: the "did you mean" text for an unknown name, if any. +pub fn selector_name_help(value: &str, candidates: Vec<&str>) -> Option { + if crate::graph::_helpers::selectors::is_pattern(value) { + return None; + } + crate::graph::_helpers::selectors::suggestion(value, candidates) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/unique_id_edges.rs b/crates/sqlbuild-analysis/src/graph/main/unique_id_edges.rs new file mode 100644 index 0000000000..6562a27685 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/unique_id_edges.rs @@ -0,0 +1,13 @@ +use std::collections::HashMap; + +use crate::graph::models::{EdgeDirection, ProjectGraph}; + +/// Each key's neighbours as `prefix.project.name` ids in graph order; unknown types use `model`. +pub fn unique_id_edges( + graph: &ProjectGraph, + direction: EdgeDirection, + project_name: &str, + prefixes: &HashMap, +) -> Vec<(String, Vec)> { + crate::graph::_helpers::unique_ids::unique_id_edges(graph, direction, project_name, prefixes) +} diff --git a/crates/sqlbuild-analysis/src/graph/mod.rs b/crates/sqlbuild-analysis/src/graph/mod.rs new file mode 100644 index 0000000000..1bb62c4b82 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/mod.rs @@ -0,0 +1,10 @@ +//! The compiled project graph: dependency indexes and selector resolution. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/graph/models.rs b/crates/sqlbuild-analysis/src/graph/models.rs new file mode 100644 index 0000000000..e13685f2e6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/models.rs @@ -0,0 +1,84 @@ +//! The project graph, the resources it is built from and selector failures. + +use std::collections::HashMap; + +use crate::assembly::project::types::ObjectKey; + +/// One graph resource in project order: its key, upstream keys, tags and model folder. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GraphResource { + pub key: ObjectKey, + pub deps: Vec, + pub tags: Vec, + /// A model's directory relative to the project, `models/` prefix optional; `None` otherwise. + pub folder: Option, +} + +/// Python's `ProjectGraph` indexes, each in Python's dict insertion order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ProjectGraph { + /// Lineage edges with SQL tests stripped. + pub upstream: Vec<(ObjectKey, Vec)>, + /// Inverted edges, each list sorted by `(resource type, name)`. + pub downstream: Vec<(ObjectKey, Vec)>, + /// Tag to keys, keys in first-tagged order. + pub tags: Vec<(String, Vec)>, + /// Model key to its folder below `models/`. + pub paths: Vec<(ObjectKey, String)>, + /// Selector name to key; a later resource with the same name replaces the key in place. + pub names: Vec<(String, ObjectKey)>, + pub(crate) upstream_index: HashMap, + pub(crate) downstream_index: HashMap, + pub(crate) name_index: HashMap, +} + +/// One parsed selector token: `kind:value` (kind `name` when bare) or `start~end`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ParsedSelector { + Kind { + kind: String, + value: String, + upstream: bool, + downstream: bool, + }, + Path { + start: String, + end: String, + upstream: bool, + downstream: bool, + }, +} + +/// Which resources `expand_required_build_resources` adds around a selected scope. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BuildResources { + pub upstream_functions: bool, + pub upstream_seeds: bool, + pub downstream_functions: bool, +} + +impl BuildResources { + /// What `--select`/`--exclude` resolution adds: upstream functions only. + pub const SELECTION: Self = Self { + upstream_functions: true, + upstream_seeds: false, + downstream_functions: false, + }; +} + +/// Python's `ProjectGraph` indexes as handed in, in Python's dict order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct GraphIndexes { + pub names: Vec<(String, ObjectKey)>, + pub upstream: Vec<(ObjectKey, Vec)>, + pub downstream: Vec<(ObjectKey, Vec)>, + pub tags: Vec<(String, Vec)>, + pub paths: Vec<(ObjectKey, String)>, +} + +/// Which edge direction a graph query follows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EdgeDirection { + Upstream, + Downstream, +} diff --git a/crates/sqlbuild-analysis/src/graph/tests.rs b/crates/sqlbuild-analysis/src/graph/tests.rs new file mode 100644 index 0000000000..a9d1498b0f --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_graph.rs"] +mod graph; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/graph/tests/helpers.rs b/crates/sqlbuild-analysis/src/graph/tests/helpers.rs new file mode 100644 index 0000000000..78a99a86be --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/tests/helpers.rs @@ -0,0 +1,89 @@ +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::main::build_project_graph::build_project_graph; +use crate::graph::main::resolve_selectors::resolve_selectors; +use crate::graph::models::{GraphResource, ProjectGraph}; +use crate::graph::tests::test_types::SelectorFailure; + +type OwnedFailure = (String, String, Option); + +type ResourceSpec = ( + &'static str, + &'static str, + &'static [(&'static str, &'static str)], + &'static [&'static str], + Option<&'static str>, +); + +/// Models, a source, a seed and a UDF with tags and folders, in Python's project order. +const ORDERS_PROJECT: &[ResourceSpec] = &[ + ( + "model", + "orders", + &[("source", "raw_orders"), ("udf", "tax")], + &["daily"], + Some("mart"), + ), + ( + "model", + "order_items", + &[("model", "orders"), ("seed", "countries")], + &["daily", "finance"], + Some("models/mart/items"), + ), + ("model", "customers", &[], &[], Some("staging/crm")), + ( + "model", + "customer_orders", + &[("model", "customers"), ("model", "orders")], + &[], + Some("models"), + ), + ("source", "raw_orders", &[], &[], None), + ("seed", "countries", &[], &["ref"], None), + ("udf", "tax", &[], &[], None), +]; + +pub(super) fn key(pair: &(&str, &str)) -> ObjectKey { + (pair.0.to_owned(), pair.1.to_owned()) +} + +pub(super) fn orders_graph() -> ProjectGraph { + let resources: Vec = ORDERS_PROJECT + .iter() + .map(|(kind, name, deps, tags, folder)| GraphResource { + key: key(&(kind, name)), + deps: deps.iter().map(key).collect(), + tags: tags.iter().map(|tag| (*tag).to_owned()).collect(), + folder: folder.map(str::to_owned), + }) + .collect(); + build_project_graph(&resources) +} + +/// The resolved keys, or the error's `(code, message, help)`. +pub(super) fn resolved(select: &[&str], exclude: &[&str]) -> Result, OwnedFailure> { + let owned = |values: &[&str]| { + values + .iter() + .map(|value| (*value).to_owned()) + .collect::>() + }; + resolve_selectors(&orders_graph(), &owned(select), &owned(exclude)).map_err( + |SelectorError { + code, + message, + help, + }| (code.to_owned(), message, help), + ) +} + +/// The keys a selector case expects. +pub(super) fn keys(pairs: &[(&str, &str)]) -> Vec { + pairs.iter().map(key).collect() +} + +/// An owned copy of the `(code, message, help)` a selector case expects. +pub(super) fn failure((code, message, help): SelectorFailure) -> OwnedFailure { + (code.to_owned(), message.to_owned(), help.map(str::to_owned)) +} diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs b/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs new file mode 100644 index 0000000000..5a68adc08b --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/tests/test_graph.rs @@ -0,0 +1,349 @@ +use crate::graph::_helpers::close_matches::close_matches; +use crate::graph::tests::helpers::{failure, keys, resolved}; +use crate::graph::tests::test_types::{ + CloseMatchTestCase, SelectorErrorTestCase, SelectorTestCase, +}; + +#[test] +fn given_selectors_when_resolving_then_keys_match_python() { + let test_cases = [ + SelectorTestCase { + description: "nothing selected", + select: &[], + exclude: &[], + expected_keys: &[ + ("model", "customer_orders"), + ("model", "customers"), + ("model", "order_items"), + ("model", "orders"), + ("seed", "countries"), + ("source", "raw_orders"), + ("udf", "tax"), + ], + }, + SelectorTestCase { + description: "orders", + select: &["orders"], + exclude: &[], + expected_keys: &[("model", "orders"), ("udf", "tax")], + }, + SelectorTestCase { + description: "+orders", + select: &["+orders"], + exclude: &[], + expected_keys: &[ + ("model", "orders"), + ("source", "raw_orders"), + ("udf", "tax"), + ], + }, + SelectorTestCase { + description: "orders+", + select: &["orders+"], + exclude: &[], + expected_keys: &[ + ("model", "customer_orders"), + ("model", "order_items"), + ("model", "orders"), + ("udf", "tax"), + ], + }, + SelectorTestCase { + description: "tag:daily minus order_items", + select: &["tag:daily"], + exclude: &["order_items"], + expected_keys: &[("model", "orders"), ("udf", "tax")], + }, + SelectorTestCase { + description: "models/mart", + select: &["models/mart"], + exclude: &[], + expected_keys: &[ + ("model", "order_items"), + ("model", "orders"), + ("udf", "tax"), + ], + }, + SelectorTestCase { + description: "path:models/staging", + select: &["path:models/staging"], + exclude: &[], + expected_keys: &[("model", "customers")], + }, + SelectorTestCase { + description: "order*", + select: &["order*"], + exclude: &[], + expected_keys: &[ + ("model", "order_items"), + ("model", "orders"), + ("udf", "tax"), + ], + }, + SelectorTestCase { + description: "seed:countries+", + select: &["seed:countries+"], + exclude: &[], + expected_keys: &[ + ("model", "order_items"), + ("seed", "countries"), + ("udf", "tax"), + ], + }, + SelectorTestCase { + description: "customers~customer_orders", + select: &["customers~customer_orders"], + exclude: &[], + expected_keys: &[ + ("model", "customer_orders"), + ("model", "customers"), + ("udf", "tax"), + ], + }, + SelectorTestCase { + description: "+orders~order_items+", + select: &["+orders~order_items+"], + exclude: &[], + expected_keys: &[ + ("model", "order_items"), + ("model", "orders"), + ("source", "raw_orders"), + ("udf", "tax"), + ], + }, + SelectorTestCase { + description: "tag:daily,models/mart/items", + select: &["tag:daily,models/mart/items"], + exclude: &[], + expected_keys: &[("model", "order_items"), ("udf", "tax")], + }, + SelectorTestCase { + description: "orders customers", + select: &["orders customers"], + exclude: &[], + expected_keys: &[("model", "customers"), ("model", "orders"), ("udf", "tax")], + }, + SelectorTestCase { + description: " minus orders", + select: &[], + exclude: &["orders"], + expected_keys: &[ + ("model", "customer_orders"), + ("model", "customers"), + ("model", "order_items"), + ("seed", "countries"), + ("source", "raw_orders"), + ("udf", "tax"), + ], + }, + SelectorTestCase { + description: "source:raw_*", + select: &["source:raw_*"], + exclude: &[], + expected_keys: &[("source", "raw_orders")], + }, + SelectorTestCase { + description: "[!o]*", + select: &["[!o]*"], + exclude: &[], + expected_keys: &[ + ("model", "customer_orders"), + ("model", "customers"), + ("seed", "countries"), + ("source", "raw_orders"), + ("udf", "tax"), + ], + }, + ]; + + for test_case in test_cases { + assert_eq!( + resolved(test_case.select, test_case.exclude), + Ok(keys(test_case.expected_keys)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_selectors_when_resolving_then_errors_match_python() { + let test_cases = [ + SelectorErrorTestCase { + description: "models", + select: &["models"], + exclude: &[], + expected_error: ("S007", "unknown selector name 'models'", None), + }, + SelectorErrorTestCase { + description: "ordrs", + select: &["ordrs"], + exclude: &[], + expected_error: ( + "S007", + "unknown selector name 'ordrs'", + Some("did you mean 'orders'?"), + ), + }, + SelectorErrorTestCase { + description: "ord*x", + select: &["ord*x"], + exclude: &[], + expected_error: ("S007", "unknown selector pattern 'ord*x'", None), + }, + SelectorErrorTestCase { + description: "a+b", + select: &["a+b"], + exclude: &[], + expected_error: ( + "S002", + "selector 'a+b' contains '+' in an unsupported position", + None, + ), + }, + SelectorErrorTestCase { + description: "+", + select: &["+"], + exclude: &[], + expected_error: ( + "S004", + "selector '+' has no name after removing '+' markers", + None, + ), + }, + SelectorErrorTestCase { + description: "a,,b", + select: &["a,,b"], + exclude: &[], + expected_error: ("S007", "unknown selector name 'a'", None), + }, + SelectorErrorTestCase { + description: "x~", + select: &["x~"], + exclude: &[], + expected_error: ( + "S003", + "path selector 'x~' requires names on both sides of '~'", + None, + ), + }, + SelectorErrorTestCase { + description: "foo:bar", + select: &["foo:bar"], + exclude: &[], + expected_error: ("S005", "unknown selector type 'foo' in 'foo:bar'", None), + }, + SelectorErrorTestCase { + description: "seed:", + select: &["seed:"], + exclude: &[], + expected_error: ("S006", "selector 'seed:' has empty value after ':'", None), + }, + SelectorErrorTestCase { + description: "tag:none", + select: &["tag:none"], + exclude: &[], + expected_error: ("S008", "no models found with tag 'none'", None), + }, + SelectorErrorTestCase { + description: "staging/crm.sql", + select: &["staging/crm.sql"], + exclude: &[], + expected_error: ( + "S012", + "path selectors require an explicit root: use 'models/' or 'python/'", + None, + ), + }, + SelectorErrorTestCase { + description: "mart", + select: &["mart"], + exclude: &[], + expected_error: ("S007", "unknown selector name 'mart'", None), + }, + SelectorErrorTestCase { + description: "test:abc", + select: &["test:abc"], + exclude: &[], + expected_error: ( + "S013", + "selector 'test:abc' selects a unit test; only `sqb test` and `sqb build` accept unit-test selectors", + None, + ), + }, + SelectorErrorTestCase { + description: "task:abc", + select: &["task:abc"], + exclude: &[], + expected_error: ( + "S010", + "selector type 'task' does not map to a resource type yet", + None, + ), + }, + SelectorErrorTestCase { + description: "order_items~orders", + select: &["order_items~orders"], + exclude: &[], + expected_error: ( + "S000", + "'model:orders' is not downstream of 'model:order_items'", + None, + ), + }, + SelectorErrorTestCase { + description: "customer", + select: &["customer"], + exclude: &[], + expected_error: ( + "S007", + "unknown selector name 'customer'", + Some("did you mean 'customers'?"), + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + resolved(test_case.select, test_case.exclude), + Err(failure(test_case.expected_error)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_unknown_name_when_suggesting_then_matches_get_close_matches() { + let test_cases = [ + CloseMatchTestCase { + description: "close matches for ordrs", + word: "ordrs", + candidates: &[ + "customer_orders", + "customers", + "order_items", + "orders", + "raw_orders", + ], + cutoff: 0.8, + expected_matches: &["orders"], + }, + CloseMatchTestCase { + description: "close matches for custmer", + word: "custmer", + candidates: &["costumer", "custom", "customer", "customers"], + cutoff: 0.6, + expected_matches: &["customer", "customers", "costumer"], + }, + ]; + + for test_case in test_cases { + assert_eq!( + close_matches(test_case.word, test_case.candidates, 3, test_case.cutoff), + test_case.expected_matches, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/graph/tests/test_types.rs b/crates/sqlbuild-analysis/src/graph/tests/test_types.rs new file mode 100644 index 0000000000..affff90bf4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/tests/test_types.rs @@ -0,0 +1,24 @@ +pub(super) struct SelectorTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) exclude: &'static [&'static str], + pub(super) expected_keys: &'static [(&'static str, &'static str)], +} + +pub(super) struct SelectorErrorTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) exclude: &'static [&'static str], + pub(super) expected_error: SelectorFailure, +} + +/// `(code, message, help)` of the `PlannerInputError` Python raises. +pub(super) type SelectorFailure = (&'static str, &'static str, Option<&'static str>); + +pub(super) struct CloseMatchTestCase { + pub(super) description: &'static str, + pub(super) word: &'static str, + pub(super) candidates: &'static [&'static str], + pub(super) cutoff: f64, + pub(super) expected_matches: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 4d62706cbb..42d1a928f3 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -3,16 +3,21 @@ #![forbid(unsafe_code)] pub mod assembly; +pub mod binding_locations; pub mod column_references; +pub mod compiled_project; pub mod compiler; mod constants; pub mod contracts; pub mod explicit_references; +pub mod graph; pub mod lineage; pub mod query_analysis; pub mod relation_names; +pub mod resource_diagnostics; pub mod semantic_checks; pub mod semantic_usage; pub mod semantic_validation; +pub mod sql_test_attachment; pub mod sql_tokens; pub mod type_system; diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/authored_files.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/authored_files.rs new file mode 100644 index 0000000000..1928fa6454 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/authored_files.rs @@ -0,0 +1,90 @@ +//! Python's `_authored_files`: `sorted(rglob("*"))` minus exclusions, over discovery's snapshot. + +use std::path::{Path, PathBuf}; + +use sqlbuild_discovery::models::StageFailure; +use sqlbuild_discovery::tree::main::interrupted_listings::interrupted_listings; +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::{ProjectTree, TreeEntry}; + +use crate::lineage::constants::{ + FINGERPRINT_EXCLUDED_PART, FINGERPRINT_EXCLUDED_ROOTS, FINGERPRINT_ROOT_FILES, + FINGERPRINT_SUFFIXES, +}; + +/// One hashed file: its path and its `as_posix()` project-relative path. +pub(crate) struct AuthoredFile { + pub(crate) path: PathBuf, + pub(crate) relative_path: String, +} + +impl AuthoredFile { + /// `PurePath.suffix`, lowercased as `str.lower` does. + pub(crate) fn lower_suffix(&self) -> String { + let name = self.relative_path.rsplit('/').next().unwrap_or(""); + python_suffix(name).to_lowercase() + } +} + +/// The files Python hashes, in its order; an undecodable path makes Python's encode fail. +pub(crate) enum AuthoredFiles { + /// The files, and whether any walked directory's listing failed after it opened. + Files { + files: Vec, + interrupted: bool, + }, + /// A hashed path whose name is not valid UTF-8 (or UTF-16): Python returns `None`. + Undecodable, + /// Discovery's walk could not run; it lists unreadable directories as empty. + Unavailable(StageFailure), +} + +/// `sorted(rglob("*"))`: no links to directories, unreadable directories empty, `PurePath` order. +pub(crate) fn authored_files(project_dir: &Path) -> AuthoredFiles { + let tree = ProjectTree::new(project_dir); + let paths = match rglob(&tree, "", is_hashed_entry) { + Ok(paths) => paths, + Err(failure) => return AuthoredFiles::Unavailable(failure), + }; + let mut files: Vec = Vec::new(); + for relative_path in paths { + if is_excluded(&relative_path) || !tree.absolute(&relative_path).is_file() { + continue; + } + if tree.is_undecodable(&relative_path) { + return AuthoredFiles::Undecodable; + } + files.push(AuthoredFile { + path: tree.absolute(&relative_path), + relative_path, + }); + } + AuthoredFiles::Files { + files, + interrupted: !interrupted_listings(&tree).is_empty(), + } +} + +fn is_hashed_entry(entry: &TreeEntry) -> bool { + FINGERPRINT_ROOT_FILES.contains(&entry.name.as_str()) + || FINGERPRINT_SUFFIXES.contains(&python_suffix(&entry.name).to_lowercase().as_str()) +} + +/// `parts[0] in _EXCLUDED_ROOTS` or `"__pycache__" in parts`. +fn is_excluded(relative_path: &str) -> bool { + let mut parts = relative_path.split('/'); + parts + .next() + .is_some_and(|first| FINGERPRINT_EXCLUDED_ROOTS.contains(&first)) + || relative_path + .split('/') + .any(|part| part == FINGERPRINT_EXCLUDED_PART) +} + +/// `PurePath.suffix`: from the last dot, unless the dot starts or ends the name. +fn python_suffix(name: &str) -> &str { + match name.rfind('.') { + Some(index) if index > 0 && index < name.len() - 1 => &name[index..], + _ => "", + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/environment_markers.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/environment_markers.rs new file mode 100644 index 0000000000..53c880e57f --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/environment_markers.rs @@ -0,0 +1,73 @@ +//! Python's `ENV:\s*([A-Za-z0-9_]+)` scan over bytes, with its cacheability checks. + +use crate::lineage::constants::{ENVIRONMENT_MARKER, NON_ASCII_BYTE_START}; + +/// The names `contents` reads; `None` for a nameless marker or a name before a non-ASCII byte. +pub(crate) fn environment_names(contents: &[u8]) -> Option> { + let mut names: Vec<&str> = Vec::new(); + for start in marker_positions(contents, true) { + let Some(end) = match_end(contents, start) else { + continue; + }; + if contents + .get(end) + .is_some_and(|byte| *byte >= NON_ASCII_BYTE_START) + { + return None; + } + let name = &contents[start + ENVIRONMENT_MARKER.len()..end]; + let Ok(name) = std::str::from_utf8(name) else { + return None; + }; + names.push(name.trim_start_matches(is_python_space_char)); + } + (names.len() == marker_positions(contents, false).len()).then_some(names) +} + +fn is_python_space_char(character: char) -> bool { + u8::try_from(character).is_ok_and(|byte| is_python_bytes_space(&byte)) +} + +/// Non-overlapping marker starts (`bytes.count`); `matching` skips past each regex match instead. +fn marker_positions(contents: &[u8], matching: bool) -> Vec { + let mut positions: Vec = Vec::new(); + let mut position = 0; + while let Some(offset) = find(&contents[position..], ENVIRONMENT_MARKER) { + let start = position + offset; + positions.push(start); + position = if matching { + match_end(contents, start).unwrap_or(start + 1) + } else { + start + ENVIRONMENT_MARKER.len() + }; + } + positions +} + +/// Where a regex match starting at `start` ends, if it matches. +fn match_end(contents: &[u8], start: usize) -> Option { + let mut position = start + ENVIRONMENT_MARKER.len(); + while contents.get(position).is_some_and(is_python_bytes_space) { + position += 1; + } + let name_start = position; + while contents.get(position).is_some_and(is_name_byte) { + position += 1; + } + (position > name_start).then_some(position) +} + +fn find(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +/// Python's bytes-pattern `\s`: ASCII space, tab, newline, return, vertical tab and form feed. +fn is_python_bytes_space(byte: &u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c) +} + +fn is_name_byte(byte: &u8) -> bool { + byte.is_ascii_alphanumeric() || *byte == b'_' +} diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/fingerprint_digest.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/fingerprint_digest.rs new file mode 100644 index 0000000000..7d4b7d16c9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/fingerprint_digest.rs @@ -0,0 +1,113 @@ +//! The digest updates of Python's `relation_lineage_fingerprint`, after its prefix. + +use std::collections::BTreeSet; +use std::fmt::Write; + +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use sha2::{Digest, Sha256}; +use sqlbuild_discovery::models::StageFailure; + +use crate::lineage::_helpers::authored_files::{AuthoredFile, AuthoredFiles}; +use crate::lineage::_helpers::environment_markers::environment_names; +use crate::lineage::constants::{ + DYNAMIC_CONTEXT_MARKER, DYNAMIC_CONTEXT_SUFFIXES, FINGERPRINT_SCAN_BATCH_SIZE, + MISSING_ENVIRONMENT_VALUE, +}; +use crate::lineage::models::{InterruptedListingPolicy, RelationFingerprint}; + +struct ScannedFile { + contents: Vec, + environment: Vec, +} + +/// The fingerprint of the walked files under `policy` for interrupted listings. +pub(crate) fn fingerprint_outcome( + authored: AuthoredFiles, + prefix: &[u8], + policy: InterruptedListingPolicy, +) -> RelationFingerprint { + match authored { + AuthoredFiles::Files { + interrupted: true, .. + } if policy == InterruptedListingPolicy::Uncacheable => RelationFingerprint::Uncacheable, + AuthoredFiles::Files { files, .. } => fingerprint_digest(&files, prefix).map_or( + RelationFingerprint::Uncacheable, + RelationFingerprint::Digest, + ), + AuthoredFiles::Undecodable => RelationFingerprint::Uncacheable, + AuthoredFiles::Unavailable(StageFailure::Unlistable { .. }) => { + RelationFingerprint::Uncacheable + } + AuthoredFiles::Unavailable(StageFailure::Internal(message)) => { + RelationFingerprint::Failed(message) + } + } +} + +/// The hex digest, or `None` where Python returns `None`. +fn fingerprint_digest(files: &[AuthoredFile], prefix: &[u8]) -> Option { + let mut digest = Sha256::new(); + digest.update(prefix); + let mut environment: BTreeSet = BTreeSet::new(); + for batch in files.chunks(FINGERPRINT_SCAN_BATCH_SIZE) { + let scanned: Vec> = batch.par_iter().map(scanned_file).collect(); + for (file, scanned) in batch.iter().zip(scanned) { + let scanned = scanned?; + environment.extend(scanned.environment); + digest.update((file.relative_path.chars().count() as u64).to_be_bytes()); + digest.update(file.relative_path.as_bytes()); + digest.update((scanned.contents.len() as u64).to_be_bytes()); + digest.update(&scanned.contents); + } + } + for name in &environment { + digest.update(name.as_bytes()); + digest.update(environment_value(name)?.as_bytes()); + } + Some( + digest + .finalize() + .iter() + .fold(String::new(), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }), + ) +} + +/// `os.environ.get(name, "")`; `None` where its UTF-8 encoding fails. +fn environment_value(name: &str) -> Option { + let Some(value) = std::env::var_os(name) else { + return Some(MISSING_ENVIRONMENT_VALUE.to_owned()); + }; + let Ok(value) = value.into_string() else { + return None; + }; + Some(value) +} + +/// One file's bytes and environment names; `None` where Python returns `None`. +fn scanned_file(file: &AuthoredFile) -> Option { + let Ok(contents) = std::fs::read(&file.path) else { + return None; + }; + if contains(&contents, DYNAMIC_CONTEXT_MARKER) + && DYNAMIC_CONTEXT_SUFFIXES.contains(&file.lower_suffix().as_str()) + { + return None; + } + let environment: Vec = environment_names(&contents)? + .into_iter() + .map(str::to_owned) + .collect(); + Some(ScannedFile { + contents, + environment, + }) +} + +fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) +} diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs index b9f011aba7..139d0d6c87 100644 --- a/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/mod.rs @@ -1,5 +1,10 @@ +pub(crate) mod authored_files; +pub(crate) mod environment_markers; +pub(crate) mod fingerprint_digest; pub(crate) mod parsed_lineage; pub(crate) mod projections; pub(crate) mod python_payload; pub(crate) mod references; +pub(crate) mod rich_lineage; +pub(crate) mod semantic_uses; pub(crate) mod stars; diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs new file mode 100644 index 0000000000..302db0fdb6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/rich_lineage.rs @@ -0,0 +1,294 @@ +//! Python's `_build_polyglot_model_column_lineage`: rich lineage from polyglot query analysis. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use polyglot_sql::{ + AnalyzeQueryOptions, ColumnReferenceFact, ComplexityGuardOptions, DialectType, ProjectionFact, + ProjectionNullability, QueryAnalysis, ReferenceConfidence, SchemaColumn, SchemaTable, + TransformKind, ValidationSchema, analyze_query, +}; + +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::lineage::_helpers::references::{ + PhysicalResource, normalized_sql, physical_resource_name, physical_resources, +}; +use crate::lineage::_helpers::stars::star_lineage; +use crate::lineage::constants::{MAX_FUNCTION_CALL_DEPTH, STAR_COLUMN_NAME, UNKNOWN_COLUMN_TYPE}; +use crate::lineage::models::{ + LineageColumn, LineageConfidence, LineageNullability, LineageResourceType, + LineageSchemaResource, LineageSource, LineageTransformKind, RichLineageColumn, + RichLineageOutcome, RichSchemaResource, +}; + +/// What every model of one request shares: the dialect, the guard and the known tables. +pub(crate) struct RichContext { + dialect: DialectType, + guard: ComplexityGuardOptions, + /// `_polyglot_schema_tables`: each physical name's columns, sorted by name. + tables: HashMap, + /// `_build_schema_mapping`'s column names, in its insertion order, for star expansion. + names: HashMap>, +} + +impl RichContext { + /// Only tables some model references are built: the wheel is only ever sent those. + pub(crate) fn new( + dialect: DialectType, + schema: &[RichSchemaResource], + referenced: &HashSet, + names: HashMap>, + ) -> Self { + Self { + dialect, + guard: ComplexityGuardOptions { + max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), + ..ComplexityGuardOptions::default() + }, + tables: schema_tables(schema, referenced), + names, + } + } +} + +/// Analyse every model in parallel; the first panic in request order is the request's error. +pub(crate) fn outcomes_in_order( + models: &[String], + analyse: impl Fn(&str) -> RichLineageOutcome + Sync, +) -> Result, String> { + let outcomes: Vec> = models + .par_iter() + .map(|query_sql| catch_compiler_panic(|| Ok(analyse(query_sql)))) + .collect(); + outcomes.into_iter().collect() +} + +/// The column names star expansion reads: assigned then defaulted, first position kept. +pub(crate) fn schema_names(resource: &RichSchemaResource) -> LineageSchemaResource { + LineageSchemaResource { + resource_type: resource.resource_type, + name: resource.name.clone(), + columns: resource + .assigned + .iter() + .chain(&resource.defaulted) + .map(|(name, _)| name.clone()) + .collect(), + } +} + +/// The physical name's typed columns; a later resource with columns replaces an earlier one. +pub(crate) fn schema_tables( + schema: &[RichSchemaResource], + referenced: &HashSet, +) -> HashMap { + let mut tables: HashMap = HashMap::with_capacity(referenced.len()); + for resource in schema { + let physical_name = physical_resource_name(resource.resource_type, &resource.name); + if !referenced.contains(&physical_name) { + continue; + } + let mut types: HashMap<&str, &str> = HashMap::new(); + for (name, column_type) in &resource.assigned { + let _ = types.insert(name, python_type(column_type.as_deref())); + } + for (name, column_type) in &resource.defaulted { + let _ = types + .entry(name) + .or_insert_with(|| python_type(column_type.as_deref())); + } + if types.is_empty() { + continue; + } + let mut columns: Vec<(&str, &str)> = types.into_iter().collect(); + columns.sort_unstable(); + let table = SchemaTable { + name: physical_name.clone(), + schema: None, + columns: columns + .into_iter() + .map(|(name, column_type)| SchemaColumn { + name: name.to_owned(), + data_type: column_type.to_owned(), + nullable: None, + primary_key: false, + unique: false, + references: None, + }) + .collect(), + aliases: Vec::new(), + primary_key: Vec::new(), + unique_keys: Vec::new(), + foreign_keys: Vec::new(), + }; + let _ = tables.insert(physical_name, table); + } + tables +} + +/// Python's `type or "UNKNOWN"`. +fn python_type(column_type: Option<&str>) -> &str { + column_type + .filter(|column_type| !column_type.is_empty()) + .unwrap_or(UNKNOWN_COLUMN_TYPE) +} + +pub(crate) fn rich_model_lineage(query_sql: &str, context: &RichContext) -> RichLineageOutcome { + let dialect = context.dialect; + let physical = physical_resources(query_sql); + let referenced: BTreeSet<&str> = physical + .iter() + .map(|resource| resource.physical_name.as_str()) + .collect(); + let options = AnalyzeQueryOptions { + complexity_guard: Some(context.guard), + dialect, + schema: Some(ValidationSchema { + tables: referenced + .into_iter() + .filter_map(|name| context.tables.get(name).cloned()) + .collect(), + strict: None, + }), + }; + let analysis: QueryAnalysis = match analyze_query(&normalized_sql(query_sql), options) { + Ok(analysis) => analysis, + Err(error) => return RichLineageOutcome::Skipped(error.to_string()), + }; + let resource_by_physical_name: HashMap<&str, &PhysicalResource> = physical + .iter() + .map(|resource| (resource.physical_name.as_str(), resource)) + .collect(); + let star_expanded: HashSet<&str> = analysis + .star_projections + .iter() + .flat_map(|star| &star.expanded_columns) + .filter(|column| !column.is_empty()) + .map(String::as_str) + .collect(); + let mut columns: Vec = Vec::new(); + for projection in &analysis.projections { + if projection.is_star { + continue; + } + let Some(output_column) = projection.name.as_deref().filter(|name| !name.is_empty()) else { + continue; + }; + if star_expanded.contains(output_column) { + continue; + } + let (upstream_columns, confidence) = + projection_upstreams(&projection.upstream, &resource_by_physical_name); + let transform_kind = + transform_kind(projection.transform_kind, !upstream_columns.is_empty()); + let confidence = + if !upstream_columns.is_empty() || transform_kind == LineageTransformKind::Constant { + confidence + } else { + LineageConfidence::Unknown + }; + columns.push(RichLineageColumn { + column: LineageColumn { + output_column: output_column.to_owned(), + transform_kind, + confidence, + upstream_columns, + }, + nullability: nullability(projection), + }); + } + let has_star = !star_expanded.is_empty(); + if has_star { + let star_columns = star_lineage( + &context.names, + &physical, + columns + .iter() + .map(|column| column.column.output_column.as_str()), + ); + columns.extend(star_columns.into_iter().map(|column| RichLineageColumn { + column, + nullability: LineageNullability::Unknown, + })); + } + RichLineageOutcome::Built { columns, has_star } +} + +/// `_projection_upstreams`, including its order: a later unaliased guess overwrites UNKNOWN. +fn projection_upstreams( + upstream: &[ColumnReferenceFact], + resource_by_physical_name: &HashMap<&str, &PhysicalResource>, +) -> (Vec, LineageConfidence) { + let mut sources: Vec = Vec::new(); + let mut seen: HashSet<(LineageResourceType, &str, &str)> = HashSet::new(); + let mut confidence = LineageConfidence::High; + for reference in upstream { + let column = reference.column.as_str(); + if column.is_empty() || column == STAR_COLUMN_NAME { + continue; + } + let source_name = reference + .source_name + .as_deref() + .filter(|name| !name.is_empty()) + .or(reference.table.as_deref()) + .filter(|name| !name.is_empty()); + let Some(source_name) = source_name else { + confidence = LineageConfidence::Unknown; + continue; + }; + let Some(resource) = resource_by_physical_name.get(source_name) else { + continue; + }; + if reference.confidence != ReferenceConfidence::Resolved && reference.source_alias.is_none() + { + confidence = LineageConfidence::Medium; + } + if !seen.insert(( + resource.resource_type, + resource.resource_name.as_str(), + column, + )) { + continue; + } + sources.push(LineageSource { + resource_type: resource.resource_type, + resource_name: resource.resource_name.clone(), + column_name: column.to_owned(), + }); + } + sources.sort_by(|left, right| { + ( + left.resource_type.as_str(), + &left.resource_name, + &left.column_name, + ) + .cmp(&( + right.resource_type.as_str(), + &right.resource_name, + &right.column_name, + )) + }); + (sources, confidence) +} + +/// `_projection_transform_kind`: polyglot's `star` reads as an expression or a constant. +fn transform_kind(kind: TransformKind, has_upstream: bool) -> LineageTransformKind { + match kind { + TransformKind::Cast => LineageTransformKind::Cast, + TransformKind::Aggregation => LineageTransformKind::Aggregation, + TransformKind::Constant => LineageTransformKind::Constant, + _ if !has_upstream => LineageTransformKind::Constant, + TransformKind::Direct => LineageTransformKind::Direct, + TransformKind::Expression | TransformKind::Star => LineageTransformKind::Expression, + } +} + +fn nullability(projection: &ProjectionFact) -> LineageNullability { + match projection.nullability { + ProjectionNullability::NonNull => LineageNullability::NonNull, + ProjectionNullability::Nullable => LineageNullability::Nullable, + ProjectionNullability::Unknown => LineageNullability::Unknown, + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/_helpers/semantic_uses.rs b/crates/sqlbuild-analysis/src/lineage/_helpers/semantic_uses.rs new file mode 100644 index 0000000000..92edbfda46 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/_helpers/semantic_uses.rs @@ -0,0 +1,154 @@ +//! Direct semantic uses resolved to SQLBuild resources and authored locations. + +use std::collections::BTreeMap; + +use serde_json::{Value, json}; + +use crate::lineage::_helpers::references::{ + normalized_sql, physical_resource_name, physical_resources, +}; +use crate::lineage::models::{DirectSemanticUse, RichSchemaResource}; + +/// Keep resource precedence and model order identical to the compiler's lineage view. +pub(crate) fn direct_semantic_uses( + schema: &[RichSchemaResource], + models: &[(String, String, String)], + dialect: &str, +) -> Result, String> { + let tables: Vec = schema_mapping(schema) + .into_iter() + .map(schema_table) + .collect(); + model_uses(models, dialect, &tables) +} + +/// Assigned columns override earlier values; declared defaults never override inference. +fn schema_mapping(schema: &[RichSchemaResource]) -> BTreeMap> { + let mut tables: BTreeMap> = BTreeMap::new(); + for resource in schema { + let mut columns: BTreeMap = BTreeMap::new(); + for (name, kind) in &resource.assigned { + columns.insert(name.clone(), column_type(kind)); + } + for (name, kind) in &resource.defaulted { + columns + .entry(name.clone()) + .or_insert_with(|| column_type(kind)); + } + if !columns.is_empty() { + tables.insert( + physical_resource_name(resource.resource_type, &resource.name), + columns, + ); + } + } + tables +} + +fn model_uses( + models: &[(String, String, String)], + dialect: &str, + tables: &[Value], +) -> Result, String> { + let mut result: Vec = Vec::new(); + for (model, query, authored) in models { + let resources = physical_resources(query); + let request = json!({ + "sql": normalized_sql(query), "dialect": dialect, + "schema": {"strict": true, "tables": tables} + }); + let response = crate::semantic_usage::main::analyze_json(&request.to_string())?; + let response: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let Some(uses) = response.get("uses").and_then(Value::as_array) else { + return Err("native semantic usage analysis returned an invalid response".to_owned()); + }; + for usage in uses { + let (Some(source), Some(column), Some(context), Some(expression)) = ( + usage.get("sourceName").and_then(Value::as_str), + usage.get("column").and_then(Value::as_str), + usage.get("context").and_then(Value::as_str), + usage.get("expressionSql").and_then(Value::as_str), + ) else { + continue; + }; + let Some(resource) = resources + .iter() + .rev() + .find(|item| item.physical_name == source) + else { + continue; + }; + let fallback = ( + usage.get("line").and_then(Value::as_u64), + usage.get("columnOffset").and_then(Value::as_u64), + ); + let (line, offset) = authored_position(authored, column).unwrap_or(fallback); + result.push(DirectSemanticUse { + model: model.clone(), + context: context.to_owned(), + expression: expression.to_owned(), + resource_type: resource.resource_type, + resource_name: resource.resource_name.clone(), + column: column.to_owned(), + confidence: usage + .get("confidence") + .and_then(Value::as_str) + .filter(|value| !value.is_empty()) + .unwrap_or("unknown") + .to_owned(), + line, + offset, + }); + } + } + Ok(result) +} + +fn column_type(value: &Option) -> String { + value + .as_deref() + .filter(|value| !value.is_empty()) + .unwrap_or("UNKNOWN") + .to_owned() +} + +fn schema_table((name, columns): (String, BTreeMap)) -> Value { + let columns: Vec = columns + .into_iter() + .map(|(name, kind)| json!({"name": name, "type": kind})) + .collect(); + json!({"name": name, "columns": columns}) +} + +fn authored_position(sql: &str, column: &str) -> Option<(Option, Option)> { + let word = |value: char| value.is_ascii_alphanumeric() || value == '_'; + let mut next_allowed = 0; + let mut matches = sql.char_indices().filter(|(start, _)| { + if *start < next_allowed + || !sql[*start..].starts_with(column) + || sql[..*start].chars().next_back().is_some_and(word) + || sql[*start + column.len()..] + .chars() + .next() + .is_some_and(word) + { + return false; + } + next_allowed = *start + column.len(); + true + }); + let (start, _) = matches.next()?; + if matches.next().is_some() { + return None; + } + let prefix = &sql[..start]; + let line = prefix.bytes().filter(|byte| *byte == b'\n').count() as u64 + 1; + let offset = prefix + .rsplit('\n') + .next() + .unwrap_or_default() + .chars() + .count() as u64 + + 1; + Some((Some(line), Some(offset))) +} diff --git a/crates/sqlbuild-analysis/src/lineage/constants.rs b/crates/sqlbuild-analysis/src/lineage/constants.rs index 10e8c04e7b..cf37260374 100644 --- a/crates/sqlbuild-analysis/src/lineage/constants.rs +++ b/crates/sqlbuild-analysis/src/lineage/constants.rs @@ -27,3 +27,36 @@ pub(crate) const REFERENCE_PREFIX: &str = "__"; pub(crate) const UDF_CALL: &str = "__udf("; pub(crate) const PHYSICAL_RESOURCE_PREFIX: &str = "__sqlbuild_"; pub(crate) const PHYSICAL_NAME_REPLACEMENT: &str = "__"; +/// The column type Python's schema mapping gives a column without one. +pub(crate) const UNKNOWN_COLUMN_TYPE: &str = "UNKNOWN"; +/// Rich lineage workers, matching the compile's analysis pool. +pub(crate) const RICH_LINEAGE_WORKERS: usize = 4; +/// Polyglot analysis recurses deeply on long set-operation chains. +pub(crate) const RICH_LINEAGE_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; +/// Suffixes (lowercased) of the authored files the relation lineage fingerprint hashes. +pub(crate) const FINGERPRINT_SUFFIXES: &[&str] = &[".csv", ".py", ".sql", ".toml", ".yaml", ".yml"]; +/// File names hashed wherever they appear, whatever their suffix. +pub(crate) const FINGERPRINT_ROOT_FILES: &[&str] = &[".gitignore", ".sqlbuildignore"]; +/// Bound retained file contents while preserving parallel reads and ordered hashing. +pub(crate) const FINGERPRINT_SCAN_BATCH_SIZE: usize = 64; +/// Top-level entries the fingerprint never reads. +pub(crate) const FINGERPRINT_EXCLUDED_ROOTS: &[&str] = &[ + ".git", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".tox", + ".venv", + "node_modules", + "target", + "venv", +]; +/// A path part excluded at any depth. +pub(crate) const FINGERPRINT_EXCLUDED_PART: &str = "__pycache__"; +pub(crate) const ENVIRONMENT_MARKER: &[u8] = b"ENV:"; +pub(crate) const DYNAMIC_CONTEXT_MARKER: &[u8] = b"CTX:"; +/// Suffixes whose dynamic context marker makes the graph uncacheable. +pub(crate) const DYNAMIC_CONTEXT_SUFFIXES: &[&str] = &[".py", ".toml"]; +pub(crate) const MISSING_ENVIRONMENT_VALUE: &str = ""; +/// A name directly followed by a byte from here on makes the graph uncacheable. +pub(crate) const NON_ASCII_BYTE_START: u8 = 128; diff --git a/crates/sqlbuild-analysis/src/lineage/main/build_rich_lineage.rs b/crates/sqlbuild-analysis/src/lineage/main/build_rich_lineage.rs new file mode 100644 index 0000000000..ef1aa4adc3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/main/build_rich_lineage.rs @@ -0,0 +1,63 @@ +//! Rich column lineage: Python's polyglot `analyze_query` path, answered natively. + +use std::collections::HashSet; + +use polyglot_sql::DialectType; + +use crate::lineage::_helpers::references::{physical_resource_name, physical_resources}; +use crate::lineage::_helpers::rich_lineage::{ + RichContext, outcomes_in_order, rich_model_lineage, schema_names, +}; +use crate::lineage::_helpers::stars::schema_mapping; +use crate::lineage::constants::{RICH_LINEAGE_WORKER_STACK_BYTES, RICH_LINEAGE_WORKERS}; +use crate::lineage::main::parser_dialect::parser_dialect; +use crate::lineage::models::{LineageSchemaResource, RichLineageOutcome, RichLineageRequest}; + +/// One outcome per model, in request order; a parser panic fails the whole request. +pub fn build_rich_lineage(request: &RichLineageRequest) -> Result, String> { + if request.models.is_empty() { + return Ok(Vec::new()); + } + let Ok(dialect) = parser_dialect(Some(&request.dialect)) else { + return Err(format!("Unknown dialect: {}", request.dialect)); + }; + let context = request_context(request, dialect); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(RICH_LINEAGE_WORKERS.min(request.models.len())) + .stack_size(RICH_LINEAGE_WORKER_STACK_BYTES) + .thread_name(|index| format!("sqlbuild-rich-lineage-{index}")) + .build() + .map_err(|error| error.to_string())?; + pool.install(|| { + outcomes_in_order(&request.models, |query_sql| { + rich_model_lineage(query_sql, &context) + }) + }) +} + +/// The shared context: only tables some model references are built. +fn request_context(request: &RichLineageRequest, dialect: DialectType) -> RichContext { + let referenced: HashSet = request + .models + .iter() + .flat_map(|query_sql| physical_resources(query_sql)) + .map(|resource| resource.physical_name) + .collect(); + let names: Vec = request + .schema + .iter() + .map(schema_names) + .filter(|resource| { + referenced.contains(&physical_resource_name( + resource.resource_type, + &resource.name, + )) + }) + .collect(); + RichContext::new( + dialect, + &request.schema, + &referenced, + schema_mapping(&names), + ) +} diff --git a/crates/sqlbuild-analysis/src/lineage/main/direct_semantic_uses.rs b/crates/sqlbuild-analysis/src/lineage/main/direct_semantic_uses.rs new file mode 100644 index 0000000000..c06b04ace0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/main/direct_semantic_uses.rs @@ -0,0 +1,12 @@ +//! Direct semantic lineage uses over native model and schema facts. + +use crate::lineage::models::{DirectSemanticUse, RichSchemaResource}; + +/// Resolve semantic uses to project resources and authored character positions. +pub fn direct_semantic_uses( + schema: &[RichSchemaResource], + models: &[(String, String, String)], + dialect: &str, +) -> Result, String> { + crate::lineage::_helpers::semantic_uses::direct_semantic_uses(schema, models, dialect) +} diff --git a/crates/sqlbuild-analysis/src/lineage/main/mod.rs b/crates/sqlbuild-analysis/src/lineage/main/mod.rs index 6b75bd22ea..356cac0968 100644 --- a/crates/sqlbuild-analysis/src/lineage/main/mod.rs +++ b/crates/sqlbuild-analysis/src/lineage/main/mod.rs @@ -1,2 +1,5 @@ pub mod build_fast_lineage; +pub mod build_rich_lineage; +pub mod direct_semantic_uses; pub mod parser_dialect; +pub mod relation_fingerprint; diff --git a/crates/sqlbuild-analysis/src/lineage/main/relation_fingerprint.rs b/crates/sqlbuild-analysis/src/lineage/main/relation_fingerprint.rs new file mode 100644 index 0000000000..7faf54411f --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/main/relation_fingerprint.rs @@ -0,0 +1,16 @@ +//! Python's `relation_lineage_fingerprint` over the authored files, after its prefix. + +use std::path::Path; + +use crate::lineage::_helpers::authored_files::authored_files; +use crate::lineage::_helpers::fingerprint_digest::fingerprint_outcome; +use crate::lineage::models::{InterruptedListingPolicy, RelationFingerprint}; + +/// Hash Python's encoded `prefix`, the authored files and the environment values they read. +pub fn relation_fingerprint( + project_dir: &Path, + prefix: &[u8], + policy: InterruptedListingPolicy, +) -> RelationFingerprint { + fingerprint_outcome(authored_files(project_dir), prefix, policy) +} diff --git a/crates/sqlbuild-analysis/src/lineage/models.rs b/crates/sqlbuild-analysis/src/lineage/models.rs index f4821c90fb..f660fbad29 100644 --- a/crates/sqlbuild-analysis/src/lineage/models.rs +++ b/crates/sqlbuild-analysis/src/lineage/models.rs @@ -8,6 +8,20 @@ pub enum LineageResourceType { Seed, } +/// A semantic column use projected onto a project resource and authored SQL. +#[derive(Debug)] +pub struct DirectSemanticUse { + pub model: String, + pub context: String, + pub expression: String, + pub resource_type: LineageResourceType, + pub resource_name: String, + pub column: String, + pub confidence: String, + pub line: Option, + pub offset: Option, +} + impl LineageResourceType { /// Python's `CompiledResourceType` value. pub fn as_str(self) -> &'static str { @@ -147,3 +161,98 @@ impl FastLineageOutcome { } } } + +/// Python's `InferredNullability` values that rich lineage produces. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LineageNullability { + NonNull, + Nullable, + Unknown, +} + +impl LineageNullability { + pub fn as_str(self) -> &'static str { + match self { + Self::NonNull => "non_null", + Self::Nullable => "nullable", + Self::Unknown => "unknown", + } + } +} + +/// One resource's typed columns, as Python's schema mapping builds them. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RichSchemaResource { + pub resource_type: LineageResourceType, + pub name: String, + /// `columns[name] = type or "UNKNOWN"`: a repeat keeps its position and takes the later type. + pub assigned: Vec<(String, Option)>, + /// `columns.setdefault(name, type or "UNKNOWN")`, applied after `assigned`. + pub defaulted: Vec<(String, Option)>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RichLineageRequest { + /// The `PolyglotAnalysisDialect` value Python resolved from the adapter's dialect. + pub dialect: String, + pub schema: Vec, + /// Each requested model's compiled query SQL, in project order. + pub models: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RichLineageColumn { + pub column: LineageColumn, + pub nullability: LineageNullability, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RichLineageOutcome { + Built { + columns: Vec, + has_star: bool, + }, + /// Polyglot rejected the query; Python logs this message and records no lineage. + Skipped(String), +} + +impl RichLineageOutcome { + /// `(status, columns, has_star, detail)`: detail is the polyglot error. + pub fn into_parts(self) -> (&'static str, Vec, bool, Option) { + match self { + Self::Built { columns, has_star } => ("built", columns, has_star, None), + Self::Skipped(message) => ("skipped", Vec::new(), false, Some(message)), + } + } +} + +/// What a directory whose `readdir` fails mid-listing does to the fingerprint, by Python version. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InterruptedListingPolicy { + /// Python 3.12: `Path.walk` lets the `OSError` escape `rglob`, so the result is `None`. + Uncacheable, + /// Python 3.13 and 3.14: `glob` skips the directory's entries. + Skip, +} + +/// The relation lineage cache key Python's `relation_lineage_fingerprint` computes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RelationFingerprint { + /// The hex SHA-256 digest of the authored inputs. + Digest(String), + /// Python returns `None`: dynamic context, malformed environment markers or unreadable input. + Uncacheable, + /// Discovery's walk failed internally: a native compiler failure, not a Python `None`. + Failed(String), +} + +impl RelationFingerprint { + /// The digest, `None` where Python returns `None`, or the internal failure. + pub fn into_digest(self) -> Result, String> { + match self { + Self::Digest(digest) => Ok(Some(digest)), + Self::Uncacheable => Ok(None), + Self::Failed(message) => Err(message), + } + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/tests.rs b/crates/sqlbuild-analysis/src/lineage/tests.rs index 191a850e53..17a909d4f5 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests.rs @@ -1,8 +1,20 @@ #[path = "tests/test_build_fast_lineage.rs"] mod build_fast_lineage; +#[path = "tests/test_build_rich_lineage.rs"] +mod build_rich_lineage; +#[path = "tests/test_direct_semantic_uses.rs"] +mod direct_semantic_uses; +#[path = "tests/test_environment_markers.rs"] +mod environment_markers; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_interrupted_listing_policy.rs"] +mod interrupted_listing_policy; #[path = "tests/test_references.rs"] mod references; +#[path = "tests/test_rich_dialects.rs"] +mod rich_dialects; +#[path = "tests/test_rich_lineage_panics.rs"] +mod rich_lineage_panics; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs index a2876a6ac4..73e206171a 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs @@ -1,9 +1,11 @@ use polyglot_sql::{DialectType, SchemaValidationOptions}; use crate::lineage::main::build_fast_lineage::build_fast_lineage; +use crate::lineage::main::build_rich_lineage::build_rich_lineage; +use crate::lineage::models::RelationFingerprint; use crate::lineage::models::{ FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, LineageResourceType, - LineageSchemaResource, + LineageSchemaResource, RichLineageColumn, RichLineageRequest, RichSchemaResource, }; use crate::semantic_validation::models::ProjectCatalog; @@ -124,3 +126,98 @@ fn resource( columns: strings(columns), } } + +/// Build one model against a fixed typed schema: `(status, column lines, has_star, detail)`. +pub(crate) fn rich_outcome( + dialect: &str, + sql: &str, +) -> (&'static str, Vec, bool, Option) { + let request = RichLineageRequest { + dialect: dialect.to_owned(), + schema: rich_schema(), + models: vec![sql.to_owned()], + }; + let mut outcomes = build_rich_lineage(&request).expect("the rich lineage pool builds"); + assert_eq!(outcomes.len(), 1); + let (status, columns, has_star, detail) = outcomes.remove(0).into_parts(); + ( + status, + columns.iter().map(described_rich_column).collect(), + has_star, + detail, + ) +} + +fn described_rich_column(column: &RichLineageColumn) -> String { + let upstream: Vec = column + .column + .upstream_columns + .iter() + .map(|source| { + format!( + "{}:{}.{}", + source.resource_type.as_str(), + source.resource_name, + source.column_name + ) + }) + .collect(); + format!( + "{} {} {} {} [{}]", + column.column.output_column, + column.column.transform_kind.as_str(), + column.column.confidence.as_str(), + column.nullability.as_str(), + upstream.join(" ") + ) +} + +/// The typed schema `wheel_cases.py` gave the wheel, spelled through assigned and defaulted columns. +fn rich_schema() -> Vec { + vec![ + RichSchemaResource { + resource_type: LineageResourceType::Model, + name: "orders".to_owned(), + assigned: typed(&[ + ("order_id", Some("INTEGER")), + ("customer_id", Some("TEXT")), + ("customer_id", Some("INTEGER")), + ("amount", Some("DOUBLE")), + ]), + defaulted: typed(&[("amount", Some("VARCHAR")), ("order_id", None)]), + }, + RichSchemaResource { + resource_type: LineageResourceType::Model, + name: "customers".to_owned(), + assigned: typed(&[("customer_id", Some("INTEGER"))]), + defaulted: typed(&[("name", Some(""))]), + }, + RichSchemaResource { + resource_type: LineageResourceType::Model, + name: "empty".to_owned(), + assigned: Vec::new(), + defaulted: Vec::new(), + }, + RichSchemaResource { + resource_type: LineageResourceType::Seed, + name: "regions".to_owned(), + assigned: typed(&[("region_id", Some("INTEGER")), ("label", Some("VARCHAR"))]), + defaulted: Vec::new(), + }, + ] +} + +fn typed(columns: &[(&str, Option<&str>)]) -> Vec<(String, Option)> { + columns + .iter() + .map(|(name, column_type)| ((*name).to_owned(), column_type.map(str::to_owned))) + .collect() +} + +/// `digest`, `uncacheable` or `failed: ` for a fingerprint outcome. +pub(crate) fn fingerprint_status(outcome: RelationFingerprint) -> String { + outcome.into_digest().map_or_else( + |message| format!("failed: {message}"), + |digest| digest.map_or("uncacheable", |_| "digest").to_owned(), + ) +} diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_build_rich_lineage.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_build_rich_lineage.rs new file mode 100644 index 0000000000..2d33d99f45 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_build_rich_lineage.rs @@ -0,0 +1,133 @@ +use crate::lineage::tests::helpers::rich_outcome; +use crate::lineage::tests::test_types::RichLineageTestCase; + +/// Expected lines are the wheel's output for the same SQL and schema (Python's rich path). +#[test] +fn given_models_when_building_rich_lineage_then_matches_the_wheel_path() { + let test_cases = [ + RichLineageTestCase { + description: "joins, aggregation, cast, constant and expression columns", + dialect: "generic", + sql: "SELECT o.order_id, c.name AS customer_name, SUM(o.amount) AS total, \ + CAST(o.amount AS BIGINT) AS amount_big, 1 AS one, o.amount + c.customer_id AS mix \ + FROM __ref('orders') o JOIN __ref('customers') c ON o.customer_id = c.customer_id \ + GROUP BY 1, 2", + expected_status: "built", + expected_lines: &[ + "order_id direct high unknown [model:orders.order_id]", + "customer_name direct high unknown [model:customers.name]", + "total aggregation high unknown [model:orders.amount]", + "amount_big cast high unknown [model:orders.amount]", + "one constant high non_null []", + "mix expression high unknown [model:customers.customer_id model:orders.amount]", + ], + expected_has_star: false, + expected_detail: None, + }, + RichLineageTestCase { + description: "a root star expands schema columns after explicit outputs", + dialect: "generic", + sql: "SELECT *, amount AS amt FROM __ref('orders')", + expected_status: "built", + expected_lines: &[ + "amt direct high unknown [model:orders.amount]", + "order_id star medium unknown [model:orders.order_id]", + "customer_id star medium unknown [model:orders.customer_id]", + "amount star medium unknown [model:orders.amount]", + ], + expected_has_star: true, + expected_detail: None, + }, + RichLineageTestCase { + description: "a CTE star over a join with a seed, under DuckDB", + dialect: "duckdb", + sql: "WITH t AS (SELECT o.*, r.label FROM __ref('orders') o \ + LEFT JOIN __seed('regions') r ON o.customer_id = r.region_id) \ + SELECT order_id, COALESCE(label, 'none') AS label, amount FROM t", + expected_status: "built", + expected_lines: &[ + "order_id direct high unknown [model:orders.order_id]", + "label expression high non_null [seed:regions.label]", + "amount direct high unknown [model:orders.amount]", + ], + expected_has_star: false, + expected_detail: None, + }, + RichLineageTestCase { + description: "set operation branches merge sorted upstreams", + dialect: "generic", + sql: "SELECT customer_id, amount FROM __ref('orders') UNION ALL \ + SELECT customer_id, 0 FROM __ref('customers')", + expected_status: "built", + expected_lines: &[ + "customer_id direct high unknown \ + [model:customers.customer_id model:orders.customer_id]", + "amount direct high unknown [model:orders.amount]", + ], + expected_has_star: false, + expected_detail: None, + }, + RichLineageTestCase { + description: "an unknown qualifier and an unresolved column", + dialect: "generic", + sql: "SELECT missing.thing, order_id FROM __ref('orders') o \ + JOIN __ref('customers') c ON TRUE", + expected_status: "built", + expected_lines: &[ + "thing constant high unknown []", + "order_id constant unknown unknown []", + ], + expected_has_star: false, + expected_detail: None, + }, + RichLineageTestCase { + description: "a resource without known columns", + dialect: "generic", + sql: "SELECT x FROM __source('unknown_feed')", + expected_status: "built", + expected_lines: &["x direct high unknown [source:unknown_feed.x]"], + expected_has_star: false, + expected_detail: None, + }, + RichLineageTestCase { + description: "a parse error skips the model with the wheel's message", + dialect: "generic", + sql: "SELECT FROM WHERE", + expected_status: "skipped", + expected_lines: &[], + expected_has_star: false, + expected_detail: Some( + "Parse error at line 1, column 18: Expected table name or subquery, got Where", + ), + }, + RichLineageTestCase { + description: "a dialect outside the old seven-dialect build is answered natively", + dialect: "mysql", + sql: "SELECT order_id FROM __ref('orders')", + expected_status: "built", + expected_lines: &["order_id direct high unknown [model:orders.order_id]"], + expected_has_star: false, + expected_detail: None, + }, + ]; + for test_case in test_cases { + let (status, lines, has_star, detail) = rich_outcome(test_case.dialect, test_case.sql); + assert_eq!( + status, test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!(lines, test_case.expected_lines, "{}", test_case.description); + assert_eq!( + has_star, test_case.expected_has_star, + "{}", + test_case.description + ); + assert_eq!( + detail.as_deref(), + test_case.expected_detail, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_direct_semantic_uses.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_direct_semantic_uses.rs new file mode 100644 index 0000000000..4df9afdd6f --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_direct_semantic_uses.rs @@ -0,0 +1,111 @@ +use crate::lineage::main::direct_semantic_uses::direct_semantic_uses; +use crate::lineage::models::{LineageResourceType, RichSchemaResource}; +use crate::lineage::tests::test_types::{SchemaFilterTestCase, SemanticUseTestCase}; + +#[test] +fn given_unrelated_tables_when_building_rich_schema_then_only_referenced_tables_are_built() { + let test_cases = [ + SchemaFilterTestCase { + description: "no references", + referenced: 0, + expected_tables: 0, + }, + SchemaFilterTestCase { + description: "one upstream", + referenced: 1, + expected_tables: 1, + }, + SchemaFilterTestCase { + description: "three upstreams", + referenced: 3, + expected_tables: 3, + }, + ]; + for test_case in test_cases { + let schema: Vec = (0..254) + .map(|index| RichSchemaResource { + resource_type: LineageResourceType::Model, + name: format!("orders_{index}"), + assigned: vec![("amount".to_owned(), Some("BIGINT".to_owned()))], + defaulted: Vec::new(), + }) + .collect(); + let expected: std::collections::HashSet = (0..test_case.referenced) + .map(|value| format!("__sqlbuild_model__orders_{value}")) + .collect(); + let tables = crate::lineage::_helpers::rich_lineage::schema_tables(&schema, &expected); + assert_eq!( + tables.len(), + test_case.expected_tables, + "{}", + test_case.description + ); + assert_eq!( + tables + .keys() + .cloned() + .collect::>(), + expected, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_authored_unicode_when_analyzing_semantic_uses_then_locations_and_resources_are_preserved() +{ + let test_cases = [SemanticUseTestCase { + description: "Unicode before a unique predicate column", + query: "SELECT 'é' AS note FROM __ref('orders') WHERE amount > 0", + expected_count: 1, + expected_line: 3, + expected_offset: 47, + }]; + for test_case in test_cases { + let schema = vec![RichSchemaResource { + resource_type: LineageResourceType::Model, + name: "orders".to_owned(), + assigned: vec![("amount".to_owned(), Some("BIGINT".to_owned()))], + defaulted: Vec::new(), + }]; + let models = vec![( + "filtered_orders".to_owned(), + test_case.query.to_owned(), + format!( + "MODEL (description 'Filtered orders');\n\n{}", + test_case.query + ), + )]; + let uses = direct_semantic_uses(&schema, &models, "duckdb").expect("valid query"); + assert_eq!( + uses.len(), + test_case.expected_count, + "{}", + test_case.description + ); + let usage = &uses[0]; + assert_eq!(usage.model, "filtered_orders", "{}", test_case.description); + assert_eq!( + usage.resource_type, + LineageResourceType::Model, + "{}", + test_case.description + ); + assert_eq!(usage.resource_name, "orders", "{}", test_case.description); + assert_eq!(usage.column, "amount", "{}", test_case.description); + assert_eq!(usage.context, "where", "{}", test_case.description); + assert_eq!( + usage.line, + Some(test_case.expected_line), + "{}", + test_case.description + ); + assert_eq!( + usage.offset, + Some(test_case.expected_offset), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_environment_markers.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_environment_markers.rs new file mode 100644 index 0000000000..9b434b56d2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_environment_markers.rs @@ -0,0 +1,52 @@ +use crate::lineage::_helpers::environment_markers::environment_names; +use crate::lineage::tests::test_types::EnvironmentMarkerTestCase; + +/// Python's `ENV:\s*([A-Za-z0-9_]+)` matches, `None` where it refuses to cache. +#[test] +fn given_environment_markers_when_scanning_then_matches_python_regex_and_checks() { + let test_cases = [ + EnvironmentMarkerTestCase { + description: "no markers", + contents: b"SELECT 1", + expected_names: Some(&[]), + }, + EnvironmentMarkerTestCase { + description: "names with spacing", + contents: b"${ENV:A} ${ENV: \t\nB_2}", + expected_names: Some(&["A", "B_2"]), + }, + EnvironmentMarkerTestCase { + description: "a marker without a name", + contents: b"ENV: ", + expected_names: None, + }, + EnvironmentMarkerTestCase { + description: "a name directly before a non-ASCII byte", + contents: b"ENV:AB\xc3\x89", + expected_names: None, + }, + EnvironmentMarkerTestCase { + description: "a marker inside a matched name", + contents: b"ENV: ENV:X", + expected_names: None, + }, + EnvironmentMarkerTestCase { + description: "a name ending the file", + contents: b"x ENV:LAST", + expected_names: Some(&["LAST"]), + }, + EnvironmentMarkerTestCase { + description: "a repeated name", + contents: b"ENV:A ENV:A", + expected_names: Some(&["A", "A"]), + }, + ]; + for test_case in test_cases { + assert_eq!( + environment_names(test_case.contents).as_deref(), + test_case.expected_names, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_interrupted_listing_policy.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_interrupted_listing_policy.rs new file mode 100644 index 0000000000..ea7b0ed102 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_interrupted_listing_policy.rs @@ -0,0 +1,82 @@ +use crate::lineage::_helpers::authored_files::AuthoredFiles; +use crate::lineage::_helpers::fingerprint_digest::fingerprint_outcome; +use crate::lineage::models::InterruptedListingPolicy; +use crate::lineage::tests::helpers::fingerprint_status; +use crate::lineage::tests::test_types::{ + InterruptedListingPolicyTestCase, UnavailableWalkTestCase, +}; +use sqlbuild_discovery::models::{ReadFailure, StageFailure}; + +/// A listing interrupted after its directory opened is uncacheable on 3.12 and skipped later. +#[test] +fn given_interrupted_listings_when_fingerprinting_then_policy_decides_cacheability() { + let test_cases = [ + InterruptedListingPolicyTestCase { + description: "Python 3.12 escapes the error from Path.walk", + interrupted: true, + policy: InterruptedListingPolicy::Uncacheable, + expected_status: "uncacheable", + }, + InterruptedListingPolicyTestCase { + description: "Python 3.13 and later skip the directory", + interrupted: true, + policy: InterruptedListingPolicy::Skip, + expected_status: "digest", + }, + InterruptedListingPolicyTestCase { + description: "no interruption under the 3.12 policy", + interrupted: false, + policy: InterruptedListingPolicy::Uncacheable, + expected_status: "digest", + }, + ]; + for test_case in test_cases { + let authored = AuthoredFiles::Files { + files: Vec::new(), + interrupted: test_case.interrupted, + }; + let status = fingerprint_status(fingerprint_outcome(authored, b"prefix", test_case.policy)); + assert_eq!( + status, test_case.expected_status, + "{}", + test_case.description + ); + } +} + +/// An unlistable root is Python's `OSError` (`None`); an internal walk failure is an error. +#[test] +fn given_an_unavailable_walk_when_fingerprinting_then_only_internal_failures_fail() { + let test_cases = [ + UnavailableWalkTestCase { + description: "an unlistable directory is uncacheable", + failure: || StageFailure::Unlistable { + relative_path: String::new(), + error: ReadFailure::Io { + errno: Some(13), + winerror: None, + message: "Permission denied".to_owned(), + }, + }, + expected_status: "uncacheable", + }, + UnavailableWalkTestCase { + description: "a walk that could not start fails", + failure: || StageFailure::Internal("pool".to_owned()), + expected_status: "failed: pool", + }, + ]; + for test_case in test_cases { + let authored = AuthoredFiles::Unavailable((test_case.failure)()); + let status = fingerprint_status(fingerprint_outcome( + authored, + b"prefix", + InterruptedListingPolicy::Skip, + )); + assert_eq!( + status, test_case.expected_status, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_rich_dialects.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_dialects.rs new file mode 100644 index 0000000000..e8ea355bc7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_dialects.rs @@ -0,0 +1,69 @@ +use polyglot_sql::DialectType; + +use crate::lineage::main::parser_dialect::parser_dialect; +use crate::lineage::tests::test_types::RichDialectTestCase; + +/// Every `PolyglotAnalysisDialect` value resolves as the wheel's serde decoding did. +#[test] +fn given_analysis_dialect_names_when_resolving_then_match_the_wheel_decoding() { + let test_cases = [ + RichDialectTestCase { + description: "every PolyglotAnalysisDialect value", + names: &[ + "generic", + "postgresql", + "mysql", + "bigquery", + "snowflake", + "duckdb", + "sqlite", + "hive", + "spark", + "trino", + "presto", + "redshift", + "tsql", + "oracle", + "clickhouse", + "databricks", + "athena", + "teradata", + "doris", + "starrocks", + "materialize", + "risingwave", + "singlestore", + "cockroachdb", + "tidb", + "druid", + "solr", + "tableau", + "dune", + "fabric", + "drill", + "dremio", + "exasol", + "datafusion", + ], + expected_known: true, + }, + RichDialectTestCase { + description: "a name no dialect carries", + names: &["orders_sql"], + expected_known: false, + }, + ]; + for test_case in test_cases { + for name in test_case.names { + let decoded: Option = + serde_json::from_value(serde_json::Value::String((*name).to_owned())).ok(); + assert_eq!(parser_dialect(Some(name)).ok(), decoded, "{name}"); + assert_eq!( + decoded.is_some(), + test_case.expected_known, + "{}: {name}", + test_case.description + ); + } + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs new file mode 100644 index 0000000000..b4a0944961 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs @@ -0,0 +1,71 @@ +use sqlbuild_core::constants::PANIC_MESSAGE; + +use crate::lineage::_helpers::rich_lineage::outcomes_in_order; +use crate::lineage::models::RichLineageOutcome; +use crate::lineage::tests::test_types::RichPanicTestCase; + +const PANICKING_SQL: &str = "SELECT secret_token FROM orders"; +const MANY_MODELS: &[&str] = &[ + "SELECT 0", + "SELECT 1", + "SELECT 2", + "SELECT 3", + "SELECT 4", + "SELECT 5", + "SELECT 6", + "SELECT 7", + "SELECT 8", + "SELECT 9", + "SELECT 10", + "SELECT 11", + "SELECT 12", + "SELECT 13", +]; + +/// A panic fails the request with the fixed native error (no SQL, no payload), never a deferral. +#[test] +fn given_an_injected_panic_when_building_rich_lineage_then_the_request_fails() { + let test_cases = [ + RichPanicTestCase { + description: "no panic answers every model in request order", + models: MANY_MODELS, + expected_outcome: Ok(MANY_MODELS), + }, + RichPanicTestCase { + description: "one panicking model fails the request", + models: &["SELECT 1", PANICKING_SQL, "SELECT 2"], + expected_outcome: Err(PANIC_MESSAGE), + }, + RichPanicTestCase { + description: "several panicking models still fail with the same error", + models: &[PANICKING_SQL, "SELECT 1", PANICKING_SQL], + expected_outcome: Err(PANIC_MESSAGE), + }, + ]; + for test_case in test_cases { + let models: Vec = test_case + .models + .iter() + .map(|sql| (*sql).to_owned()) + .collect(); + let outcomes = outcomes_in_order(&models, |query_sql| { + assert_ne!(query_sql, PANICKING_SQL, "injected parser panic"); + RichLineageOutcome::Skipped(query_sql.to_owned()) + }); + let answered: Result, String> = outcomes.map(|outcomes| { + outcomes + .into_iter() + .map(|outcome| outcome.into_parts().3.unwrap_or_default()) + .collect() + }); + assert_eq!( + answered + .as_ref() + .map(|sqls| sqls.iter().map(String::as_str).collect::>()) + .map_err(String::as_str), + test_case.expected_outcome.map(<[&str]>::to_vec), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_types.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_types.rs index 60235bf023..150d29c881 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_types.rs @@ -1,3 +1,20 @@ +use crate::lineage::models::InterruptedListingPolicy; +use sqlbuild_discovery::models::StageFailure; + +pub(crate) struct SchemaFilterTestCase { + pub(crate) description: &'static str, + pub(crate) referenced: usize, + pub(crate) expected_tables: usize, +} + +pub(crate) struct SemanticUseTestCase { + pub(crate) description: &'static str, + pub(crate) query: &'static str, + pub(crate) expected_count: usize, + pub(crate) expected_line: u64, + pub(crate) expected_offset: u64, +} + pub(crate) struct ReferenceTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, @@ -32,3 +49,47 @@ pub(crate) struct DeepUnionTestCase { pub(crate) expected_lines: &'static [&'static str], pub(crate) expected_detail: Option<&'static str>, } + +pub(crate) struct RichLineageTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_status: &'static str, + /// One `output transform confidence nullability [type:resource.column ...]` line per column. + pub(crate) expected_lines: &'static [&'static str], + pub(crate) expected_has_star: bool, + pub(crate) expected_detail: Option<&'static str>, +} + +pub(crate) struct EnvironmentMarkerTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static [u8], + /// The names read, or `None` where Python refuses to cache the graph. + pub(crate) expected_names: Option<&'static [&'static str]>, +} + +pub(crate) struct InterruptedListingPolicyTestCase { + pub(crate) description: &'static str, + pub(crate) interrupted: bool, + pub(crate) policy: InterruptedListingPolicy, + pub(crate) expected_status: &'static str, +} + +pub(crate) struct RichPanicTestCase { + pub(crate) description: &'static str, + pub(crate) models: &'static [&'static str], + /// The answered models' SQL in order, or the request's error. + pub(crate) expected_outcome: Result<&'static [&'static str], &'static str>, +} + +pub(crate) struct UnavailableWalkTestCase { + pub(crate) description: &'static str, + pub(crate) failure: fn() -> StageFailure, + pub(crate) expected_status: &'static str, +} + +pub(crate) struct RichDialectTestCase { + pub(crate) description: &'static str, + pub(crate) names: &'static [&'static str], + pub(crate) expected_known: bool, +} diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/constants.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/constants.rs new file mode 100644 index 0000000000..f487b93725 --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/constants.rs @@ -0,0 +1,10 @@ +//! Fixed texts of the compile-input resource diagnostics. + +pub(crate) const LINE_COMMENT_START: &str = "--"; +pub(crate) const BLOCK_COMMENT_START: &str = "/*"; +pub(crate) const BLOCK_COMMENT_END: &str = "*/"; +pub(crate) const UNCLOSED_BLOCK_COMMENT: &str = "SQL contains an unclosed block comment"; +pub(crate) const INCREMENTAL: &str = "incremental"; +pub(crate) const ROLLING_WINDOW: &str = "rolling_window"; +pub(crate) const RELATION_KINDS: [&str; 4] = ["ref", "source", "seed", "dbt_ref"]; +pub(crate) const CURSOR_HELP: &str = "read the data through __ref(), __source() or __seed() so its cursor can bound the window, or use materialized table if the model has no upstream data"; diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/main/cursor_lacks_inputs.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/main/cursor_lacks_inputs.rs new file mode 100644 index 0000000000..73e33f0f75 --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/main/cursor_lacks_inputs.rs @@ -0,0 +1,28 @@ +//! P011: a cursor-based incremental model that reads no input to bound its cursor window. + +use crate::resource_diagnostics::constants::{ + CURSOR_HELP, INCREMENTAL, RELATION_KINDS, ROLLING_WINDOW, +}; +use crate::resource_diagnostics::models::CursorModel; + +/// The P011 message and help for `name`, when its cursor window has no input to read. +#[must_use] +pub fn cursor_lacks_inputs(name: &str, model: &CursorModel<'_>) -> Option<(String, &'static str)> { + let lacks: bool = model.materialized == Some(INCREMENTAL) + && model.cursor.is_some() + && model.microbatch_strategy != Some(ROLLING_WINDOW) + && !model.has_cursor_inputs + && !model + .reference_kinds + .iter() + .any(|kind| RELATION_KINDS.contains(kind)); + lacks.then(|| { + ( + format!( + "incremental model '{name}' reads no inputs, so builds after the first cannot \ + work out their cursor window" + ), + CURSOR_HELP, + ) + }) +} diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/main/header_code_offset.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/main/header_code_offset.rs new file mode 100644 index 0000000000..3ba88260be --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/main/header_code_offset.rs @@ -0,0 +1,25 @@ +//! Where a SQL-authored file's statement header starts. + +use sqlbuild_core::text::main::is_python_space::is_python_space; + +use crate::resource_diagnostics::constants::{ + BLOCK_COMMENT_END, BLOCK_COMMENT_START, LINE_COMMENT_START, UNCLOSED_BLOCK_COMMENT, +}; + +/// The code-point offset of the first code after leading whitespace and comments. +pub fn header_code_offset(contents: &str) -> Result { + let mut rest: &str = contents; + loop { + rest = rest.trim_start_matches(is_python_space); + if let Some(comment) = rest.strip_prefix(LINE_COMMENT_START) { + rest = comment.find('\n').map_or("", |end| &comment[end + 1..]); + } else if rest.starts_with(BLOCK_COMMENT_START) { + let end: usize = rest[BLOCK_COMMENT_START.len()..] + .find(BLOCK_COMMENT_END) + .ok_or(UNCLOSED_BLOCK_COMMENT)?; + rest = &rest[BLOCK_COMMENT_START.len() + end + BLOCK_COMMENT_END.len()..]; + } else { + return Ok(contents.chars().count() - rest.chars().count()); + } + } +} diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/main/header_help.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/main/header_help.rs new file mode 100644 index 0000000000..83822aebcc --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/main/header_help.rs @@ -0,0 +1,24 @@ +//! The help that shows the header entry describing a SQL-authored resource. + +use crate::contracts::main::snippet_help::snippet_help; +use crate::resource_diagnostics::models::HeaderHelp; + +/// `to describe '', add this to the header in :` and its lines. +#[must_use] +pub fn header_help(help: &HeaderHelp<'_>) -> String { + let lines: Vec = if help.description_only { + vec![format!("{} ({});", help.statement, help.entry)] + } else { + vec![ + format!("{} (", help.statement), + format!(" {},", help.entry), + " ...".to_owned(), + ");".to_owned(), + ] + }; + snippet_help( + &format!("to describe {} '{}'", help.kind, help.name), + &format!("add this to the {} header in {}", help.statement, help.path), + &lines, + ) +} diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/main/missing_description.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/main/missing_description.rs new file mode 100644 index 0000000000..6805afb05e --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/main/missing_description.rs @@ -0,0 +1,14 @@ +//! The P010 message of a named resource without a non-empty description. + +use sqlbuild_core::text::main::python_strip::python_strip; + +/// ` '' has no description` or `... has an empty description`, if missing. +#[must_use] +pub fn missing_description(kind: &str, name: &str, description: Option<&str>) -> Option { + let state: &str = match description { + None => "has no description", + Some(text) if python_strip(text).is_empty() => "has an empty description", + Some(_) => return None, + }; + Some(format!("{kind} '{name}' {state}")) +} diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/main/mod.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/main/mod.rs new file mode 100644 index 0000000000..84c12dbc23 --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/main/mod.rs @@ -0,0 +1,5 @@ +pub mod cursor_lacks_inputs; +pub mod header_code_offset; +pub mod header_help; +pub mod missing_description; +pub mod yaml_help; diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/main/yaml_help.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/main/yaml_help.rs new file mode 100644 index 0000000000..2b4e5d9d35 --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/main/yaml_help.rs @@ -0,0 +1,16 @@ +//! The help that shows the YAML entry describing a declared seed or source. + +use crate::contracts::main::snippet_help::snippet_help; + +/// `to describe '', add `description` to its entry in :` and its lines. +#[must_use] +pub fn yaml_help(kind: &str, name: &str, path: &str) -> String { + snippet_help( + &format!("to describe {kind} '{name}'"), + &format!("add `description` to its entry in {path}"), + &[ + format!("- name: {name}"), + format!(" description: What one row of {name} represents"), + ], + ) +} diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/mod.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/mod.rs new file mode 100644 index 0000000000..376bfbf265 --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/mod.rs @@ -0,0 +1,8 @@ +//! Compile-input diagnostics of project resources: descriptions, cursor inputs and factories. + +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/models.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/models.rs new file mode 100644 index 0000000000..37d63d5d4d --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/models.rs @@ -0,0 +1,26 @@ +//! What the resource diagnostics read about a resource. + +/// The statement header help of a SQL-authored resource. +#[derive(Clone, Copy, Debug)] +pub struct HeaderHelp<'a> { + pub kind: &'a str, + pub name: &'a str, + /// `MODEL`, `SCENARIO`, `FUNCTION` or `HOOK`. + pub statement: &'a str, + /// The header entry to add, such as `description "..."`. + pub entry: &'a str, + /// The file's relative path in POSIX form. + pub path: &'a str, + /// Whether the header holds only the entry, as a hook header does. + pub description_only: bool, +} + +/// The config values and reference kinds the cursor-input check reads of one model. +#[derive(Clone, Copy, Debug)] +pub struct CursorModel<'a> { + pub materialized: Option<&'a str>, + pub cursor: Option<&'a str>, + pub microbatch_strategy: Option<&'a str>, + pub has_cursor_inputs: bool, + pub reference_kinds: &'a [&'a str], +} diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/tests.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/tests.rs new file mode 100644 index 0000000000..31eb7d9e97 --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_resource_diagnostics.rs"] +mod resource_diagnostics; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/tests/test_resource_diagnostics.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/tests/test_resource_diagnostics.rs new file mode 100644 index 0000000000..cca1589364 --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/tests/test_resource_diagnostics.rs @@ -0,0 +1,202 @@ +use crate::resource_diagnostics::main::cursor_lacks_inputs::cursor_lacks_inputs; +use crate::resource_diagnostics::main::header_code_offset::header_code_offset; +use crate::resource_diagnostics::main::header_help::header_help; +use crate::resource_diagnostics::main::missing_description::missing_description; +use crate::resource_diagnostics::main::yaml_help::yaml_help; +use crate::resource_diagnostics::models::{CursorModel, HeaderHelp}; +use crate::resource_diagnostics::tests::test_types::{ + CursorInputsTestCase, HeaderOffsetTestCase, HelpTestCase, MissingDescriptionTestCase, + YamlHelpTestCase, +}; + +#[test] +fn given_sql_files_when_locating_the_header_then_whitespace_and_comments_are_skipped() { + let test_cases = [ + HeaderOffsetTestCase { + description: "a header on the first line", + contents: "MODEL (name x);", + expected_offset: Ok(0), + }, + HeaderOffsetTestCase { + description: "line and block comments before the header", + contents: " -- lead\n\t/* block\n */ MODEL (x)", + expected_offset: Ok(24), + }, + HeaderOffsetTestCase { + description: "a file of only a line comment ends the scan", + contents: "-- only comment", + expected_offset: Ok(15), + }, + HeaderOffsetTestCase { + description: "Unicode whitespace counts as Python's", + contents: "\u{3000}\nFUNCTION ()", + expected_offset: Ok(2), + }, + HeaderOffsetTestCase { + description: "a line comment right after a block comment", + contents: "/* a */-- b\n HOOK ();", + expected_offset: Ok(14), + }, + HeaderOffsetTestCase { + description: "an unclosed block comment is Python's error", + contents: "/* open", + expected_offset: Err("SQL contains an unclosed block comment"), + }, + ]; + for test_case in test_cases { + assert_eq!( + header_code_offset(test_case.contents), + test_case.expected_offset, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_descriptions_when_checking_then_missing_and_blank_ones_are_reported() { + let test_cases = [ + MissingDescriptionTestCase { + description: "no description", + authored: None, + expected_message: Some("model 'orders' has no description"), + }, + MissingDescriptionTestCase { + description: "a blank description", + authored: Some(" \u{3000}\n"), + expected_message: Some("model 'orders' has an empty description"), + }, + MissingDescriptionTestCase { + description: "a description", + authored: Some("One row per order."), + expected_message: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + missing_description("model", "orders", test_case.authored).as_deref(), + test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_resources_when_building_description_help_then_snippets_match_python() { + let test_cases = [ + HelpTestCase { + description: "a MODEL header snippet", + help: HeaderHelp { + kind: "model", + name: "orders", + statement: "MODEL", + entry: "description \"What one row of orders represents\"", + path: "models/orders.sql", + description_only: false, + }, + expected_help: "to describe model 'orders', add this to the MODEL header in models/orders.sql:\n MODEL (\n description \"What one row of orders represents\",\n ...\n );", + }, + HelpTestCase { + description: "a one-line HOOK header snippet", + help: HeaderHelp { + kind: "hook", + name: "grant", + statement: "HOOK", + entry: "description \"What grant does\"", + path: "hooks/grant.sql", + description_only: true, + }, + expected_help: "to describe hook 'grant', add this to the HOOK header in hooks/grant.sql:\n HOOK (description \"What grant does\");", + }, + ]; + for test_case in test_cases { + assert_eq!( + header_help(&test_case.help), + test_case.expected_help, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_yaml_entries_when_building_description_help_then_snippets_match_python() { + let test_cases = [YamlHelpTestCase { + description: "a seed entry snippet", + entry: ("seed", "customers", "seeds/schema.yml"), + expected_help: "to describe seed 'customers', add `description` to its entry in seeds/schema.yml:\n - name: customers\n description: What one row of customers represents", + }]; + for test_case in test_cases { + let (kind, name, path) = test_case.entry; + + assert_eq!( + yaml_help(kind, name, path), + test_case.expected_help, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_cursor_models_when_checking_inputs_then_only_unbounded_windows_are_reported() { + let test_cases = [ + CursorInputsTestCase { + description: "an incremental cursor model reading nothing", + materialized: Some("incremental"), + microbatch_strategy: None, + has_cursor_inputs: false, + reference_kinds: &["udf", "table_fn"], + expected_lacks: true, + }, + CursorInputsTestCase { + description: "reading a source bounds the window", + materialized: Some("incremental"), + microbatch_strategy: None, + has_cursor_inputs: false, + reference_kinds: &["source"], + expected_lacks: false, + }, + CursorInputsTestCase { + description: "declared cursor inputs bound the window", + materialized: Some("incremental"), + microbatch_strategy: None, + has_cursor_inputs: true, + reference_kinds: &[], + expected_lacks: false, + }, + CursorInputsTestCase { + description: "a rolling window needs no inputs", + materialized: Some("incremental"), + microbatch_strategy: Some("rolling_window"), + has_cursor_inputs: false, + reference_kinds: &[], + expected_lacks: false, + }, + CursorInputsTestCase { + description: "a table has no cursor window", + materialized: Some("table"), + microbatch_strategy: None, + has_cursor_inputs: false, + reference_kinds: &[], + expected_lacks: false, + }, + ]; + for test_case in test_cases { + let model = CursorModel { + materialized: test_case.materialized, + cursor: Some("event_time"), + microbatch_strategy: test_case.microbatch_strategy, + has_cursor_inputs: test_case.has_cursor_inputs, + reference_kinds: test_case.reference_kinds, + }; + + assert_eq!( + cursor_lacks_inputs("events", &model).is_some(), + test_case.expected_lacks, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/resource_diagnostics/tests/test_types.rs b/crates/sqlbuild-analysis/src/resource_diagnostics/tests/test_types.rs new file mode 100644 index 0000000000..ab8893cce1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/resource_diagnostics/tests/test_types.rs @@ -0,0 +1,35 @@ +use crate::resource_diagnostics::models::HeaderHelp; + +pub(crate) struct HeaderOffsetTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) expected_offset: Result, +} + +pub(crate) struct MissingDescriptionTestCase { + pub(crate) description: &'static str, + pub(crate) authored: Option<&'static str>, + pub(crate) expected_message: Option<&'static str>, +} + +pub(crate) struct CursorInputsTestCase { + pub(crate) description: &'static str, + pub(crate) materialized: Option<&'static str>, + pub(crate) microbatch_strategy: Option<&'static str>, + pub(crate) has_cursor_inputs: bool, + pub(crate) reference_kinds: &'static [&'static str], + pub(crate) expected_lacks: bool, +} + +pub(crate) struct HelpTestCase { + pub(crate) description: &'static str, + pub(crate) help: HeaderHelp<'static>, + pub(crate) expected_help: &'static str, +} + +pub(crate) struct YamlHelpTestCase { + pub(crate) description: &'static str, + /// Kind, name and path. + pub(crate) entry: (&'static str, &'static str, &'static str), + pub(crate) expected_help: &'static str, +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs index fef4cdbd5d..f1eccc28f2 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explanation/opt_outs.rs @@ -26,8 +26,25 @@ pub(crate) fn opt_out_diagnostic( location_file: Option<&str>, hidden_codes: &[&str], ) -> Result { + let path_default = location_file == Some(PROJECT_CONFIG_FILENAME); + let (message, note, help) = unneeded_opt_out("model", name, path_default, hidden_codes)?; + Ok(OptOutDiagnostic { + model, + message, + note, + help, + }) +} + +/// Python's `unneeded_opt_out_diagnostic` message, note and help for one `kind` of resource. +pub(crate) fn unneeded_opt_out( + kind: &str, + name: &str, + path_default: bool, + hidden_codes: &[&str], +) -> Result<(String, String, String), SemanticFailure> { let findings: String = finding_counts(pattern(&FINDING_PATTERNS)?, hidden_codes); - let opt_out: &str = if location_file == Some(PROJECT_CONFIG_FILENAME) { + let opt_out: &str = if path_default { "`sql_analysis = false` from this [path_defaults] entry" } else { "`sql_analysis false`" @@ -45,16 +62,15 @@ pub(crate) fn opt_out_diagnostic( {PROJECT_CONFIG_FILENAME}:\n{SETTING_SNIPPET_INDENT}[{SETTINGS_SECTION}]\n\ {SETTING_SNIPPET_INDENT}{REQUIRE_SQL_ANALYSIS_KEY} = false" ); - Ok(OptOutDiagnostic { - model, - message: format!("`sql_analysis false` is not needed for model '{name}'"), - note: format!( + Ok(( + format!("`sql_analysis false` is not needed for {kind} '{name}'"), + format!( "{PROJECT_CONFIG_FILENAME} sets [{SETTINGS_SECTION}] {REQUIRE_SQL_ANALYSIS_KEY} = true\ , which only allows `sql_analysis false` on SQL that SQLBuild cannot parse; \ - this model parses successfully." + this {kind} parses successfully." ), - help: format!("{fix}{ADDITIONAL_HELP_SEPARATOR}{setting_help}"), - }) + format!("{fix}{ADDITIONAL_HELP_SEPARATOR}{setting_help}"), + )) } /// Python's `_finding_counts`: kinds by descending count, ties in first-seen order. diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs new file mode 100644 index 0000000000..748ea8a336 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs @@ -0,0 +1,352 @@ +//! Python's `_audit_errors`: schema audits checked against the model's closed output shape. + +use std::collections::BTreeMap; + +use serde_json::Value; + +use crate::semantic_checks::_helpers::metadata_checks::positions::text_position; +use crate::semantic_checks::_helpers::resource_sql::columns::{ + binding_dialect, resolve_binding_column, +}; +use crate::semantic_checks::models::{ + AuditFinding, AuditValue, MetadataAudit, MetadataModel, SemanticFailure, +}; +use crate::semantic_validation::main::diagnostics::binding_diagnostics; +use crate::semantic_validation::main::validations_json; +use crate::semantic_validation::models::{CatalogInput, Columns, ProjectCatalog}; +use crate::semantic_validation::types::{DiagnosticRow, Relations}; + +const EXPRESSION_AUDIT: &str = "expression_is_true"; +const RELATIONSHIPS_AUDIT: &str = "relationships"; +const ACCEPTED_VALUES_AUDIT: &str = "accepted_values"; +const OUTPUT_RELATION: &str = "output"; +const UNKNOWN_TABLE_CODE: &str = "B000"; +const MISSING_TARGET_COLUMN_CODE: &str = "B300"; +const AUDIT_TYPE_CODE: &str = "B301"; +const ERROR_SEVERITY: &str = "error"; +const BINDING_SEVERITIES: [&str; 2] = ["error", "warning"]; +const DEFAULT_BINDING_MESSAGE: &str = "SQL binding failed"; +const GENERIC_DIALECT: &str = "generic"; +const REFERENCE_KINDS: [&str; 3] = ["ref", "source", "seed"]; +const QUOTES: [char; 2] = ['\'', '"']; + +/// What the audits of one model are checked against. +pub(crate) struct AuditContext<'a> { + pub(crate) dialect: Option<&'a str>, + pub(crate) quoted_ignore_case: bool, + pub(crate) shapes: &'a std::collections::HashMap<&'a str, &'a [(String, String)]>, + pub(crate) catalog: &'a ProjectCatalog, +} + +fn internal(error: String) -> SemanticFailure { + SemanticFailure::Internal(error) +} + +fn columns(shape: &[(String, String)]) -> Columns { + Columns( + shape + .iter() + .map(|(column, data_type)| (column.clone(), Some(data_type.clone()))) + .collect(), + ) +} + +/// The referenced name of a `__ref("name")`-style call starting at `start`, if one does. +fn reference_call_at(text: &[char], start: usize) -> Option { + let rest: String = text[start..].iter().collect(); + let after = rest.strip_prefix("__")?; + let mut kinds = REFERENCE_KINDS.iter(); + let after = loop { + let kind = kinds.next()?; + if let Some(after) = after.strip_prefix(kind) { + break after; + } + }; + let after = after.strip_prefix('(')?; + let mut characters = after.chars(); + if !QUOTES.contains(&characters.next()?) { + return None; + } + let body: &str = characters.as_str(); + let name_length = body.find(QUOTES).unwrap_or(body.len()); + let close = &body[name_length..]; + (name_length > 0 && close.len() > 1 && close[1..].starts_with(')')) + .then(|| body[..name_length].to_owned()) +} + +/// `__(?:ref|source|seed)\(['"]([^'"]+)['"]\)` searched in `target`: the referenced name. +fn referenced_name(target: &str) -> Option { + let text: Vec = target.chars().collect(); + (0..text.len()).find_map(|start| reference_call_at(&text, start)) +} + +fn optional_index(value: Option<&Value>) -> Option { + value.and_then(Value::as_u64).map(|number| number as usize) +} + +/// Python's `_binding_result` over one native schema validation response. +fn response_rows(response: &Value) -> Result, SemanticFailure> { + let errors = response + .get("errors") + .and_then(Value::as_array) + .ok_or_else(|| { + internal("native SQL schema validation returned an invalid response".to_owned()) + })?; + let mut rows: Vec = Vec::new(); + for error in errors { + let (Some(severity), Some(code)) = ( + error.get("severity").and_then(Value::as_str), + error.get("code").and_then(Value::as_str), + ) else { + continue; + }; + if !BINDING_SEVERITIES.contains(&severity) { + continue; + } + let message = match error.get("message") { + Some(Value::String(text)) if !text.is_empty() => text.clone(), + _ => DEFAULT_BINDING_MESSAGE.to_owned(), + }; + rows.push(( + code.to_owned(), + message, + optional_index(error.get("line")), + optional_index(error.get("column")), + optional_index(error.get("start")), + optional_index(error.get("end")), + severity.to_owned(), + )); + } + Ok(rows) +} + +/// Python's `get_schema_validations` for one request without a catalog. +fn standalone_validation( + sql: &str, + relations: Vec<(&str, Columns)>, + dialect: Option<&str>, +) -> Result, SemanticFailure> { + let dialect = binding_dialect(Some(dialect.unwrap_or(GENERIC_DIALECT))) + .unwrap_or(GENERIC_DIALECT) + .to_owned(); + let references: BTreeMap<&str, bool> = relations + .iter() + .map(|(name, columns)| (*name, !columns.0.is_empty())) + .collect(); + let catalog = ProjectCatalog::new(CatalogInput { + dialect: dialect.clone(), + quoted_ignore_case: false, + known_functions: Vec::new(), + known_types: Vec::new(), + relations: relations + .into_iter() + .map(|(name, columns)| (name.to_owned(), columns)) + .collect(), + }) + .map_err(internal)?; + let references: Vec<(String, bool)> = references + .into_iter() + .map(|(name, closed)| (name.to_owned(), closed)) + .collect(); + let payload = catalog + .validation_payload(vec![(sql.to_owned(), references, Relations::new())]) + .map_err(internal)?; + let responses: Value = serde_json::from_str(&validations_json(&payload).map_err(internal)?) + .map_err(|error| internal(error.to_string()))?; + let response = responses + .as_array() + .and_then(|responses| responses.first()) + .ok_or_else(|| { + internal("native SQL schema validation returned an invalid batch response".to_owned()) + })?; + binding_diagnostics(sql, &dialect, response_rows(response)?).map_err(internal) +} + +/// One `(code, message, severity)` finding before it is located. +type Unlocated = (String, String, String, String); + +fn expression_findings( + expression: &str, + shape: &[(String, String)], + context: &AuditContext<'_>, +) -> Result, SemanticFailure> { + let mut overrides: Relations = Relations::new(); + if !shape.is_empty() { + overrides.insert(OUTPUT_RELATION.to_owned(), columns(shape)); + } + let request = ( + format!("SELECT * FROM {OUTPUT_RELATION} WHERE {expression}"), + vec![(OUTPUT_RELATION.to_owned(), !shape.is_empty())], + overrides, + ); + let rows = context + .catalog + .binding_results(vec![request]) + .map_err(internal)? + .into_iter() + .next() + .unwrap_or_default(); + Ok(rows + .into_iter() + .filter(|row| row.0 != UNKNOWN_TABLE_CODE) + .map(|row| { + ( + row.0, + format!("{EXPRESSION_AUDIT}: {}", row.1), + row.6, + expression.to_owned(), + ) + }) + .collect()) +} + +fn typed_findings(rows: Vec, label: &str, column: &str) -> Vec { + rows.into_iter() + .map(|row| { + let code = if row.6 == ERROR_SEVERITY { + AUDIT_TYPE_CODE.to_owned() + } else { + row.0 + }; + ( + code, + format!("{label}: {}", row.1), + row.6, + column.to_owned(), + ) + }) + .collect() +} + +fn single_column(column_type: &str) -> Columns { + Columns(vec![("value".to_owned(), Some(column_type.to_owned()))]) +} + +fn relationship_findings( + audit: &MetadataAudit, + column: &str, + shape: &[(String, String)], + context: &AuditContext<'_>, +) -> Result, SemanticFailure> { + let key = referenced_name(&audit.target).unwrap_or_else(|| audit.target.clone()); + let target_shape = context.shapes.get(key.as_str()).copied(); + let resolved = resolve_binding_column( + &audit.field, + target_shape.unwrap_or(&[]), + context.dialect, + context.quoted_ignore_case, + ); + let Some(target_shape) = target_shape else { + return Ok(Vec::new()); + }; + let Some(resolved) = resolved else { + if target_shape.is_empty() { + return Ok(Vec::new()); + } + return Ok(vec![( + MISSING_TARGET_COLUMN_CODE.to_owned(), + format!("relationships target has no column '{}'", audit.field), + ERROR_SEVERITY.to_owned(), + audit.field.clone(), + )]); + }; + let Some((_, left)) = shape.iter().find(|(name, _)| name == column) else { + return Ok(Vec::new()); + }; + let right = target_shape + .iter() + .find(|(name, _)| name == resolved) + .map_or("", |(_, data_type)| data_type.as_str()); + let rows = standalone_validation( + "SELECT lhs.value = rhs.value FROM lhs CROSS JOIN rhs", + vec![("lhs", single_column(left)), ("rhs", single_column(right))], + context.dialect, + )?; + Ok(typed_findings(rows, RELATIONSHIPS_AUDIT, column)) +} + +/// Python's `_audit_literal`. +fn audit_literal(value: &AuditValue) -> Option { + match value { + AuditValue::Null => Some("NULL".to_owned()), + AuditValue::Bool(true) => Some("TRUE".to_owned()), + AuditValue::Bool(false) => Some("FALSE".to_owned()), + AuditValue::Number(text) => Some(text.clone()), + AuditValue::Text(text) => Some(format!("'{}'", text.replace('\'', "''"))), + AuditValue::Other => None, + } +} + +fn accepted_value_findings( + values: &[AuditValue], + column: &str, + column_type: &str, + context: &AuditContext<'_>, +) -> Result, SemanticFailure> { + let literals: Option> = values.iter().map(audit_literal).collect(); + let Some(literals) = literals.filter(|literals| !literals.is_empty()) else { + return Ok(Vec::new()); + }; + let rows = standalone_validation( + &format!( + "SELECT value NOT IN ({}) FROM {OUTPUT_RELATION}", + literals.join(", ") + ), + vec![(OUTPUT_RELATION, single_column(column_type))], + context.dialect, + )?; + Ok(typed_findings(rows, ACCEPTED_VALUES_AUDIT, column)) +} + +fn audit_findings( + audit: &MetadataAudit, + shape: &[(String, String)], + context: &AuditContext<'_>, +) -> Result, SemanticFailure> { + let column = audit.column.as_deref(); + let column_type = column.and_then(|column| { + shape + .iter() + .find(|(name, _)| name == column) + .map(|(_, data_type)| data_type.as_str()) + }); + match audit.definition.as_str() { + EXPRESSION_AUDIT => match &audit.expression { + Some(expression) => expression_findings(expression, shape, context), + None => Ok(Vec::new()), + }, + RELATIONSHIPS_AUDIT => match column.filter(|column| !column.is_empty()) { + Some(column) => relationship_findings(audit, column, shape, context), + None => Ok(Vec::new()), + }, + ACCEPTED_VALUES_AUDIT => match (column, column_type, &audit.values) { + (Some(column), Some(column_type), Some(values)) => { + accepted_value_findings(values, column, column_type, context) + } + _ => Ok(Vec::new()), + }, + _ => Ok(Vec::new()), + } +} + +/// Every schema audit finding of a checked model with a closed shape, in Python's order. +pub(crate) fn audit_errors( + model: &MetadataModel, + shape: &[(String, String)], + context: &AuditContext<'_>, +) -> Result, SemanticFailure> { + let mut findings: Vec = Vec::new(); + for audit in &model.audits { + for (code, message, severity, name) in audit_findings(audit, shape, context)? { + let (line, column) = text_position(&model.authored_sql, &name, 0)?; + findings.push(AuditFinding { + code, + message, + line, + column, + is_error: severity == ERROR_SEVERITY, + }); + } + } + Ok(findings) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs index 759ba69c99..5e96bc534d 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs @@ -1,4 +1,4 @@ -//! Python's `get_semantic_metadata_diagnostics`, apart from the audit and resource SQL checks. +//! Python's `get_semantic_metadata_diagnostics`, apart from the resource SQL checks. use std::collections::{HashMap, HashSet}; use std::sync::LazyLock; @@ -6,6 +6,7 @@ use std::sync::LazyLock; use regex::Regex; use crate::semantic_checks::_helpers::explanation::messages::{compiled, pattern}; +use crate::semantic_checks::_helpers::metadata_checks::audits::{AuditContext, audit_errors}; use crate::semantic_checks::_helpers::metadata_checks::families::Families; use crate::semantic_checks::_helpers::metadata_checks::function_calls::{ FunctionContext, function_errors, @@ -22,6 +23,7 @@ use crate::semantic_checks::models::{ MetadataFinding, MetadataFunction, MetadataModel, MetadataOutcome, MetadataRequest, MetadataSource, MetadataSqlTest, ModelMetadataFindings, SemanticFailure, }; +use crate::semantic_validation::models::ProjectCatalog; use crate::type_system::models::TypeFamily; static SQL_TEST_CTE: LazyLock> = @@ -41,6 +43,7 @@ type Reference<'a> = (String, &'a str, &'a [(String, String)]); /// The metadata errors of every model, source and SQL test, or the error Python raised. pub(crate) fn check_metadata( request: &MetadataRequest, + catalog: &ProjectCatalog, ) -> Result { let shapes: Shapes<'_> = request .shapes @@ -61,9 +64,15 @@ pub(crate) fn check_metadata( return_types: &return_types, families: &families, }; + let audits = AuditContext { + dialect: request.dialect.as_deref(), + quoted_ignore_case: request.quoted_ignore_case, + shapes: &shapes, + catalog, + }; let mut models: Vec = Vec::with_capacity(request.models.len()); for model in &request.models { - models.push(model_errors(model, &context)?); + models.push(model_errors(model, &context, &audits)?); } let sources = source_errors(&request.sources, &shapes)?; let sql_tests = sql_test_errors(&request.sql_tests, &shapes)?; @@ -78,6 +87,7 @@ pub(crate) fn check_metadata( fn model_errors( model: &MetadataModel, context: &FunctionContext<'_>, + audits: &AuditContext<'_>, ) -> Result { if !model.checked { return Ok(ModelMetadataFindings::default()); @@ -86,12 +96,16 @@ fn model_errors( for (code, name, message) in function_errors(model, context)? { functions.push(model_error(model, code, &name, message)?); } - let references = match context.shapes.get(model.name.as_str()) { - Some(shape) => reference_errors(model, shape, context)?, - None => Vec::new(), + let (audits, references) = match context.shapes.get(model.name.as_str()) { + Some(shape) => ( + audit_errors(model, shape, audits)?, + reference_errors(model, shape, context)?, + ), + None => (Vec::new(), Vec::new()), }; Ok(ModelMetadataFindings { functions, + audits, references, }) } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/mod.rs index 4e52b92c32..cabc6a864f 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod argument_types; +pub(crate) mod audits; pub(crate) mod checks; pub(crate) mod families; pub(crate) mod function_calls; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs index 9d1303099c..19d4b01881 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod explanation; pub(crate) mod metadata_checks; pub(crate) mod recovery; +pub(crate) mod resource_sql; pub(crate) mod sql_text; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/columns.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/columns.rs new file mode 100644 index 0000000000..0794a1da98 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/columns.rs @@ -0,0 +1,63 @@ +//! Python's `resolve_binding_column`: the shape column an authored name binds to. + +use crate::semantic_checks::_helpers::sql_text::text::{casefold, upper}; + +const QUOTE: char = '"'; +const SNOWFLAKE: &str = "snowflake"; +const CASE_SENSITIVE_DIALECTS: [&str; 2] = ["snowflake", "postgresql"]; +/// `NATIVE_DIALECT_ALIASES`. +const DIALECT_ALIASES: [(&str, &str); 3] = [ + ("postgres", "postgresql"), + ("motherduck", "duckdb"), + ("sqlserver", "tsql"), +]; +const GENERIC_DIALECT: &str = "generic"; + +/// The binding dialect Python's aliases map `dialect` to. +pub(crate) fn binding_dialect(dialect: Option<&str>) -> Option<&str> { + let name = dialect.unwrap_or(GENERIC_DIALECT); + match DIALECT_ALIASES.iter().find(|(alias, _)| *alias == name) { + Some((_, target)) => Some(target), + None => dialect, + } +} + +fn identity(value: &str, dialect: Option<&str>, ignore_quoted_case: bool) -> String { + let quoted = value.starts_with(QUOTE) && value.ends_with(QUOTE); + let raw: String = if quoted { + value + .chars() + .skip(1) + .take(value.chars().count().saturating_sub(2)) + .collect::() + .replace("\"\"", "\"") + } else { + value.to_owned() + }; + if quoted + && dialect.is_some_and(|name| CASE_SENSITIVE_DIALECTS.contains(&name)) + && !ignore_quoted_case + { + return raw; + } + if dialect == Some(SNOWFLAKE) { + upper(&raw) + } else { + casefold(&raw) + } +} + +/// The first column of `columns` that `name` binds to, if any. +pub(crate) fn resolve_binding_column<'a>( + name: &str, + columns: &'a [(String, String)], + dialect: Option<&str>, + ignore_quoted_case: bool, +) -> Option<&'a str> { + let dialect = binding_dialect(dialect); + let expected = identity(name, dialect, ignore_quoted_case); + columns + .iter() + .map(|(column, _)| column.as_str()) + .find(|column| identity(column, dialect, ignore_quoted_case) == expected) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs new file mode 100644 index 0000000000..20d792773c --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs @@ -0,0 +1,267 @@ +//! Python's `get_resource_sql_diagnostics`: bind resource SQL against the closed model shapes. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; +use sqlbuild_core::text::main::active_python_text::active_python_text; + +use crate::semantic_checks::_helpers::explanation::opt_outs::unneeded_opt_out; +use crate::semantic_checks::_helpers::resource_sql::help::{HelpContext, resource_sql_help}; +use crate::semantic_checks::_helpers::resource_sql::locations::{ + Finding, line_at, location, text_from, +}; +use crate::semantic_checks::_helpers::resource_sql::resources::{ResourceSql, resources}; +use crate::semantic_checks::models::{ + ResourceSqlDiagnostic, ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, SemanticFailure, +}; +use crate::semantic_validation::main::normalize::normalize_analysis_sql; +use crate::semantic_validation::models::{Columns, NormalizationInput, ProjectCatalog}; +use crate::semantic_validation::types::{BindingRequest, Relations}; + +const UNKNOWN_TABLE_CODE: &str = "B000"; +const UNKNOWN_TABLE_PREFIX: &str = "Unknown table "; +const SYNTAX_ERROR_CODE: &str = "P001"; +const OPT_OUT_CODE: &str = "P009"; +const ERROR_SEVERITY: &str = "error"; +const CURSOR_CONTEXT: &str = "Model SQL"; +const ANALYSIS_CURSOR_LITERAL: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; +const GENERIC_DIALECT: &str = "generic"; + +type Key = (String, String, String, i64, i64, i64); + +fn kind_label(kind: ResourceSqlKind) -> &'static str { + match kind { + ResourceSqlKind::Audit => "audit", + ResourceSqlKind::SqlTest => "SQL test", + ResourceSqlKind::Model => "model", + } +} + +fn semantic(failure: SemanticFailure) -> ResourceSqlFailure { + ResourceSqlFailure::Internal(format!("{failure:?}")) +} + +/// Python's `normalize_analysis_sql(cursor_intrinsics_analysis_sql(sql))`. +fn cleaned_sql(sql: &str, dialect: Option<&str>) -> Result { + let (analysis_sql, _) = replace_intrinsics(active_python_text(), sql, CURSOR_CONTEXT, |_| { + ANALYSIS_CURSOR_LITERAL.to_owned() + }) + .map_err(ResourceSqlFailure::CursorIntrinsic)?; + normalize_analysis_sql(NormalizationInput { + sql: analysis_sql, + dialect: dialect.unwrap_or(GENERIC_DIALECT).to_owned(), + stubs: HashMap::new(), + placeholders: HashMap::new(), + }) + .map_err(ResourceSqlFailure::Normalization) +} + +/// The binding request for `sql` over `relations`, with each closed shape as an override. +fn binding_request( + sql: &str, + relations: &BTreeSet, + shapes: &HashMap<&str, &[(String, String)]>, +) -> BindingRequest { + let mut references: Vec<(String, bool)> = Vec::with_capacity(relations.len()); + let mut overrides: Relations = Relations::new(); + for name in relations { + let columns = shapes.get(name.as_str()).copied().unwrap_or(&[]); + references.push((name.clone(), !columns.is_empty())); + if !columns.is_empty() { + overrides.insert( + name.clone(), + Columns( + columns + .iter() + .map(|(column, data_type)| (column.clone(), Some(data_type.clone()))) + .collect(), + ), + ); + } + } + (sql.to_owned(), references, overrides) +} + +/// Python's `_reported_diagnostic`: drop open-relation lookups, make parser rejections P001. +fn reported(resource: &ResourceSql, finding: &Finding) -> Option { + if finding.0 != UNKNOWN_TABLE_CODE { + return Some(finding.clone()); + } + if finding.1.starts_with(UNKNOWN_TABLE_PREFIX) { + return None; + } + Some(( + SYNTAX_ERROR_CODE.to_owned(), + format!( + "SQL syntax error in {} '{}': {}", + kind_label(resource.kind), + resource.name, + finding.1 + ), + finding.2, + finding.3, + finding.4, + finding.5, + ERROR_SEVERITY.to_owned(), + )) +} + +/// What one resource's findings are checked against. +struct Checking<'a> { + request: &'a ResourceSqlRequest, + cleaned_sql: &'a str, +} + +/// Python's `_resource_diagnostics`: located findings keyed for deduplication. +fn resource_diagnostics( + resource: &ResourceSql, + findings: &[Finding], + checking: &Checking<'_>, +) -> Result, ResourceSqlFailure> { + let mut found: Vec<(Key, ResourceSqlDiagnostic)> = Vec::new(); + for (index, native) in findings.iter().enumerate() { + let Some(finding) = reported(resource, native) else { + continue; + }; + let span = location(resource, checking.cleaned_sql, &finding)?; + let key: Key = ( + resource.path.clone(), + finding.0.clone(), + finding.1.clone(), + span.0, + span.1, + if finding.4.is_none() { + index as i64 + } else { + -1 + }, + ); + if found.iter().any(|(existing, _)| *existing == key) { + continue; + } + let authored_text = text_from(resource, span); + let authored_line = line_at(resource, span); + let help = resource_sql_help( + &finding.0, + &finding.1, + &HelpContext { + kind: resource.kind, + authored_text: &authored_text, + authored_line: &authored_line, + column: span.1, + authored_body: &resource.authored_body, + audit_definition: resource.audit_definition.as_deref(), + shapes: &checking.request.shapes, + relations: &resource.relations, + dialect: checking.request.dialect.as_deref(), + }, + ) + .map_err(semantic)?; + let message = match &resource.label { + Some(label) if finding.0 != SYNTAX_ERROR_CODE => format!("{label}: {}", finding.1), + _ => finding.1.clone(), + }; + found.push(( + key, + ResourceSqlDiagnostic { + kind: resource.kind, + resource_name: resource.name.clone(), + is_error: finding.6 == ERROR_SEVERITY, + code: finding.0.clone(), + message, + path: resource.path.clone(), + span, + note: None, + help, + }, + )); + } + Ok(found) +} + +fn opt_out_diagnostic( + resource: &ResourceSql, + (line, column): (i64, i64), + hidden: &[(Key, ResourceSqlDiagnostic)], +) -> Result { + let codes: Vec<&str> = hidden.iter().map(|(_, item)| item.code.as_str()).collect(); + let (message, note, help) = + unneeded_opt_out(kind_label(resource.kind), &resource.name, false, &codes) + .map_err(semantic)?; + Ok(ResourceSqlDiagnostic { + kind: resource.kind, + resource_name: resource.name.clone(), + is_error: true, + code: OPT_OUT_CODE.to_owned(), + message, + path: resource.path.clone(), + span: (line, column, None, None), + note: Some(note), + help: Some(help), + }) +} + +/// Every audit, SQL test and SQL hook finding, in Python's first-seen order. +pub(crate) fn resource_sql_findings( + request: &ResourceSqlRequest, + catalog: &ProjectCatalog, +) -> Result, ResourceSqlFailure> { + let checked = resources(request); + if checked.is_empty() { + return Ok(Vec::new()); + } + let dialect = request.dialect.as_deref(); + let mut normalized: HashMap<&str, String> = HashMap::new(); + for resource in &checked { + if !normalized.contains_key(resource.sql.as_str()) { + normalized.insert(&resource.sql, cleaned_sql(&resource.sql, dialect)?); + } + } + let shapes: HashMap<&str, &[(String, String)]> = request + .shapes + .iter() + .map(|(name, columns)| (name.as_str(), columns.as_slice())) + .collect(); + let mut request_index: HashMap<(&str, &BTreeSet), usize> = HashMap::new(); + let mut requests: Vec = Vec::new(); + for resource in &checked { + let cleaned = normalized[resource.sql.as_str()].as_str(); + request_index + .entry((cleaned, &resource.relations)) + .or_insert_with(|| { + requests.push(binding_request(cleaned, &resource.relations, &shapes)); + requests.len() - 1 + }); + } + let results = catalog + .binding_results(requests) + .map_err(ResourceSqlFailure::Internal)?; + let mut keys: HashSet = HashSet::new(); + let mut opt_out_keys: HashSet<(String, i64, i64)> = HashSet::new(); + let mut diagnostics: Vec = Vec::new(); + for resource in &checked { + let cleaned = normalized[resource.sql.as_str()].as_str(); + let findings = &results[request_index[&(cleaned, &resource.relations)]]; + let checking = Checking { + request, + cleaned_sql: cleaned, + }; + let found = resource_diagnostics(resource, findings, &checking)?; + match resource.opt_out { + None => { + for (key, diagnostic) in found { + if keys.insert(key) { + diagnostics.push(diagnostic); + } + } + } + Some(position) => { + let parses = !found.iter().any(|(_, item)| item.code == SYNTAX_ERROR_CODE); + if parses && opt_out_keys.insert((resource.path.clone(), position.0, position.1)) { + diagnostics.push(opt_out_diagnostic(resource, position, &found)?); + } + } + } + } + Ok(diagnostics) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs new file mode 100644 index 0000000000..10693cdcef --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs @@ -0,0 +1,133 @@ +//! Python's `_prepared_test_sql` and `_REFERENCE_CALL_PATTERN`: test SQL with empty fixtures opened. + +use crate::semantic_checks::_helpers::resource_sql::scanning::{ + literal, slice, spaces, starts_word, words, +}; + +const REFERENCE_KINDS: [&str; 3] = ["ref", "source", "seed"]; +const FIXTURE_KINDS: [&str; 4] = ["ref", "source", "seed", "expected"]; +const EMPTY_FIXTURE: &str = "__empty_fixture"; +const OPEN_EMPTY_FIXTURE_RELATION: &str = "__sqlbuild_empty_fixture"; +const QUOTES: [char; 2] = ['\'', '"']; + +/// `__(?:ref|source|seed)\(\s*['"]([^'"]+)['"]\s*\)` at `index`: the name and match end. +fn reference_call(text: &[char], index: usize) -> Option<(String, usize)> { + let after = literal(text, index, "__", false)?; + let after = REFERENCE_KINDS + .iter() + .find_map(|kind| literal(text, after, kind, false))?; + let after = spaces(text, literal(text, after, "(", false)?, false)?; + if !QUOTES.contains(text.get(after)?) { + return None; + } + let name_end = after + + 1 + + text[after + 1..] + .iter() + .take_while(|character| !QUOTES.contains(character)) + .count(); + if name_end == after + 1 || !QUOTES.contains(text.get(name_end)?) { + return None; + } + let end = literal(text, spaces(text, name_end + 1, false)?, ")", false)?; + Some((slice(text, after + 1, name_end), end)) +} + +/// `findall` of the reference call pattern over `text`. +pub(crate) fn reference_names(text: &[char]) -> Vec { + let mut names: Vec = Vec::new(); + let mut index = 0; + while index < text.len() { + match reference_call(text, index) { + Some((name, end)) => { + names.push(name); + index = end; + } + None => index += 1, + } + } + names +} + +/// `\b__empty_fixture\s*\(\s*\)` (ignoring case) at `index`: the match end. +fn empty_fixture_call(text: &[char], index: usize) -> Option { + if !starts_word(text, index) { + return None; + } + let after = spaces(text, literal(text, index, EMPTY_FIXTURE, true)?, false)?; + let after = spaces(text, literal(text, after, "(", false)?, false)?; + literal(text, after, ")", false) +} + +/// The empty fixture CTE at `index`: its prefix end, fixture name, and match end. +fn empty_fixture_cte(text: &[char], index: usize) -> Option<(usize, String, usize)> { + if !starts_word(text, index) { + return None; + } + let after = literal(text, index, "__", false)?; + let after = FIXTURE_KINDS + .iter() + .find_map(|kind| literal(text, after, kind, true))?; + let name_start = literal(text, after, "__", false)?; + let name_end = words(text, name_start)?; + let after = spaces(text, name_end, true)?; + let after = spaces(text, literal(text, after, "AS", true)?, false)?; + let after = spaces(text, literal(text, after, "(", false)?, false)?; + let after = spaces(text, literal(text, after, "SELECT", true)?, true)?; + let after = spaces(text, literal(text, after, "*", false)?, true)?; + let prefix_end = spaces(text, literal(text, after, "FROM", true)?, true)?; + let end = empty_fixture_call_body(text, prefix_end)?; + Some((prefix_end, slice(text, name_start, name_end), end)) +} + +/// `__empty_fixture\s*\(\s*\)` (ignoring case) at `index`, without a word boundary. +fn empty_fixture_call_body(text: &[char], index: usize) -> Option { + let after = spaces(text, literal(text, index, EMPTY_FIXTURE, true)?, false)?; + let after = spaces(text, literal(text, after, "(", false)?, false)?; + literal(text, after, ")", false) +} + +/// Every empty fixture CTE in `text`: start, prefix end, name and end. +fn empty_fixture_ctes(text: &[char]) -> Vec<(usize, usize, String, usize)> { + let mut found: Vec<(usize, usize, String, usize)> = Vec::new(); + let mut index = 0; + while index < text.len() { + match empty_fixture_cte(text, index) { + Some((prefix_end, name, end)) => { + found.push((index, prefix_end, name, end)); + index = end; + } + None => index += 1, + } + } + found +} + +/// Test SQL with empty fixture CTEs reading their own name and other empty fixtures opened. +pub(crate) fn prepared_test_sql(sql_body: &str) -> (String, Vec) { + let body: Vec = sql_body.chars().collect(); + let ctes = empty_fixture_ctes(&body); + let mut opened: Vec = Vec::with_capacity(body.len()); + let mut copied = 0; + for (start, prefix_end, name, end) in &ctes { + opened.extend_from_slice(&body[copied..*start]); + opened.extend_from_slice(&body[*start..*prefix_end]); + opened.extend(name.chars()); + copied = *end; + } + opened.extend_from_slice(&body[copied..]); + let mut sql = String::with_capacity(opened.len()); + let mut index = 0; + while index < opened.len() { + if let Some(end) = empty_fixture_call(&opened, index) { + sql.push_str(OPEN_EMPTY_FIXTURE_RELATION); + index = end; + } else { + sql.push(opened[index]); + index += 1; + } + } + let mut relations = reference_names(&body); + relations.extend(ctes.into_iter().map(|(_, _, name, _)| name)); + (sql, relations) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/help.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/help.rs new file mode 100644 index 0000000000..376b6074dd --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/help.rs @@ -0,0 +1,324 @@ +//! Python's `resource_sql_help`: a concrete fix quoting the authored span of a finding. + +use std::collections::BTreeSet; +use std::sync::LazyLock; + +use regex::Regex; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::is_python_word::is_python_word; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::semantic_checks::_helpers::explanation::columns::closest_column; +use crate::semantic_checks::_helpers::explanation::messages::{missing_column, pattern}; +use crate::semantic_checks::_helpers::sql_text::python_regex::{ignorecase_word, python_regex}; +use crate::semantic_checks::_helpers::sql_text::text::{casefold, upper}; +use crate::semantic_checks::models::{ResourceSqlKind, SemanticFailure}; + +const SET_OPERATION_CODE: &str = "B215"; +const COMPARISON_CODE: &str = "B217"; +const UNKNOWN_COLUMN_CODE: &str = "B002"; +const SNOWFLAKE: &str = "snowflake"; +const RELATION_PARAMETER: &str = "@relation"; +const LISTED_COLUMNS: usize = 10; +const TEXT_TYPES: [&str; 4] = ["VARCHAR", "STRING", "TEXT", "CHAR"]; +const TEMPORAL_TYPES: [&str; 2] = ["TIMESTAMP", "DATE"]; +const SET_OPERATION_KEYWORDS: [&str; 10] = [ + "UNION", + "ALL", + "EXCEPT", + "INTERSECT", + "MINUS", + "SELECT", + "FROM", + "DISTINCT", + "BY", + "NAME", +]; +const TYPE_WORD: &str = r"(?:{W}|[()])+"; +const NAME_TAIL: &str = r"(?:{W}|\$)*"; + +static SET_OPERATION_TYPES: LazyLock> = LazyLock::new(|| { + template(&format!( + r"{} (?P{{D}}+).*{} {} (?P{TYPE_WORD}), {} {} (?P{TYPE_WORD})", + ignorecase_word("column"), + ignorecase_word("accumulated"), + ignorecase_word("type"), + ignorecase_word("next"), + ignorecase_word("type"), + )) +}); +static COMPARISON_TYPES: LazyLock> = LazyLock::new(|| { + template(&format!( + r"{} (?P{TYPE_WORD}) {} (?P{TYPE_WORD})", + ignorecase_word("between"), + ignorecase_word("and"), + )) +}); +static ALIAS_NAME: LazyLock> = LazyLock::new(|| { + template(&format!( + r#"{{S}}+{}{{S}}+"?(?P{{L}}{NAME_TAIL})"?,?\z"#, + ignorecase_word("AS") + )) +}); +static CAST: LazyLock> = LazyLock::new(|| { + template(&format!( + r"\A{}\((?P.+){{S}}+{}{{S}}+(?:{{W}}|[(), ])+\)\z", + ignorecase_word("CAST"), + ignorecase_word("AS") + )) +}); +static SINGLE_OPERAND: LazyLock> = LazyLock::new(|| { + template(&format!( + r"\A(?:'(?:[^']|'')*'|[A-Za-z_]{NAME_TAIL}(?:\.[A-Za-z_]{NAME_TAIL})?)\z" + )) +}); +static PARAMETER: LazyLock> = LazyLock::new(|| template(r"@[A-Za-z_]{W}*")); + +/// Compile one of this module's Python-class pattern templates. +fn template(text: &str) -> Result { + python_regex(text) +} + +/// What one finding's help reads besides its code and message. +pub(crate) struct HelpContext<'a> { + pub(crate) kind: ResourceSqlKind, + pub(crate) authored_text: &'a str, + pub(crate) authored_line: &'a str, + pub(crate) column: i64, + pub(crate) authored_body: &'a str, + pub(crate) audit_definition: Option<&'a str>, + pub(crate) shapes: &'a [(String, Vec<(String, String)>)], + pub(crate) relations: &'a BTreeSet, + pub(crate) dialect: Option<&'a str>, +} + +fn is_text(value: &str) -> bool { + TEXT_TYPES.iter().any(|kind| value.starts_with(kind)) +} + +fn temporal_type(types: &[String; 2]) -> Option<&'static str> { + for value in types { + for kind in TEMPORAL_TYPES { + if value.starts_with(kind) { + return Some(kind); + } + } + } + None +} + +fn shape<'a>(shapes: &'a [(String, Vec<(String, String)>)], name: &str) -> &'a [(String, String)] { + shapes + .iter() + .find(|(shape, _)| shape == name) + .map_or(&[], |(_, columns)| columns.as_slice()) +} + +/// Python's `resource_sql_help`, or None to keep the generic help. +pub(crate) fn resource_sql_help( + code: &str, + message: &str, + context: &HelpContext<'_>, +) -> Result, SemanticFailure> { + if code == SET_OPERATION_CODE { + return set_operation_help(message, context); + } + if code == COMPARISON_CODE && context.kind == ResourceSqlKind::Audit { + return audit_comparison_help(message, context); + } + if code == UNKNOWN_COLUMN_CODE && context.kind == ResourceSqlKind::SqlTest { + return unknown_column_help(message, context.shapes); + } + Ok(None) +} + +fn code_points(text: &str, start: usize, end: usize) -> String { + text.chars() + .skip(start) + .take(end.saturating_sub(start)) + .collect() +} + +/// Python's `_set_operation_operand`. +fn set_operation_operand( + context: &HelpContext<'_>, + position: &str, +) -> Result { + let preceding = if context.column > 1 { + let index = (context.column - 2) as usize; + code_points(context.authored_line, index, index + 1) + } else { + String::new() + }; + let stripped = python_strip(context.authored_text).trim_end_matches(','); + let alias = pattern(&ALIAS_NAME)?; + let mut expression: String = alias.replace_all(stripped, "").into_owned(); + if let Some(captures) = pattern(&CAST)?.captures(&expression) { + expression = python_strip(&captures["inner"]).to_owned(); + } + let preceding_is_word = preceding.chars().next().is_some_and(|character| { + character == '$' || is_python_word(active_python_text(), character) + }); + if !preceding_is_word + && pattern(&SINGLE_OPERAND)?.is_match(&expression) + && !SET_OPERATION_KEYWORDS.contains(&upper(&expression).as_str()) + { + return Ok(expression); + } + Ok(format!("COLUMN{position}")) +} + +/// Python's `_model_type`: the tested relation's type for the branch column, when one matches. +fn model_type(context: &HelpContext<'_>, operand: &str) -> Result, SemanticFailure> { + let mut names: Vec = Vec::new(); + if let Some(captures) = pattern(&ALIAS_NAME)?.captures(python_strip(context.authored_text)) { + names.push(captures["name"].to_owned()); + } + names.push( + operand + .rsplit('.') + .next() + .unwrap_or(operand) + .trim_matches('"') + .to_owned(), + ); + for name in names { + let folded = casefold(&name); + let mut found: BTreeSet = BTreeSet::new(); + for relation in context.relations { + for (output, value) in shape(context.shapes, relation) { + if casefold(output) == folded && !value.is_empty() { + found.insert(upper(value.split('(').next().unwrap_or(value))); + } + } + } + if found.len() == 1 { + return Ok(found.into_iter().next()); + } + } + Ok(None) +} + +fn set_operation_help( + message: &str, + context: &HelpContext<'_>, +) -> Result, SemanticFailure> { + let Some(captures) = pattern(&SET_OPERATION_TYPES)?.captures(message) else { + return Ok(None); + }; + let position = &captures["position"]; + let left = upper(&captures["left"]); + let right = upper(&captures["right"]); + let column = set_operation_operand(context, position)?; + if context.kind != ResourceSqlKind::SqlTest { + return Ok(Some(format!( + "Cast one branch so both branches have the same type, for example \ + `CAST({column} AS {left})` for set-operation column {position}" + ))); + } + if let Some(model_type) = model_type(context, &column)? { + return Ok(Some(format!( + "Cast set-operation column {position} to the tested model's type {model_type} in \ + the expected or fixture rows, for example `CAST({column} AS {model_type})`" + ))); + } + let target = if is_text(&left) && !is_text(&right) { + right + } else { + left + }; + Ok(Some(format!( + "Cast set-operation column {position} explicitly so every branch has the same type, \ + for example `CAST({column} AS {target})` in the expected or fixture rows" + ))) +} + +/// Python's `_audit_parameter`: the parameter nearest the finding on its line, else the only one. +fn audit_parameter(context: &HelpContext<'_>) -> Result { + let parameters = pattern(&PARAMETER)?; + let line = context.authored_line; + let nearest = parameters + .find_iter(line) + .filter(|found| found.as_str() != RELATION_PARAMETER) + .map(|found| { + let start = line[..found.start()].chars().count() as i64; + ((start + 1 - context.column).abs(), found.as_str()) + }) + .fold(None::<(i64, &str)>, |best, candidate| match best { + Some(current) if current.0 <= candidate.0 => Some(current), + _ => Some(candidate), + }); + if let Some((_, parameter)) = nearest { + return Ok(parameter.to_owned()); + } + let unique: BTreeSet<&str> = parameters + .find_iter(context.authored_body) + .map(|found| found.as_str()) + .filter(|found| *found != RELATION_PARAMETER) + .collect(); + Ok(match unique.into_iter().collect::>().as_slice() { + [only] => (*only).to_owned(), + _ => "".to_owned(), + }) +} + +fn audit_comparison_help( + message: &str, + context: &HelpContext<'_>, +) -> Result, SemanticFailure> { + let Some(captures) = pattern(&COMPARISON_TYPES)?.captures(message) else { + return Ok(None); + }; + let types = [upper(&captures["left"]), upper(&captures["right"])]; + let Some(temporal) = temporal_type(&types) else { + return Ok(None); + }; + if !types.iter().any(|value| is_text(value)) { + return Ok(None); + } + let operand = audit_parameter(context)?; + let conversion = if context.dialect == Some(SNOWFLAKE) { + format!("TO_{temporal}({operand})") + } else { + format!("CAST({operand} AS {temporal})") + }; + let audit = match context.audit_definition.filter(|name| !name.is_empty()) { + Some(name) => format!("audit '{name}'"), + None => "the audit".to_owned(), + }; + Ok(Some(format!( + "Convert the text value explicitly, for example `{conversion}`, or attach {audit} to a \ + {temporal} column" + ))) +} + +fn unknown_column_help( + message: &str, + shapes: &[(String, Vec<(String, String)>)], +) -> Result, SemanticFailure> { + let Some((name, Some(table))) = missing_column(message)? else { + return Ok(None); + }; + let columns = shape(shapes, &table); + if let Some(suggestion) = closest_column(&name, columns)? { + return Ok(Some(format!("did you mean '{suggestion}'?"))); + } + let listed: Vec<&str> = columns + .iter() + .take(LISTED_COLUMNS) + .map(|(column, _)| column.as_str()) + .collect(); + let more = if columns.len() > LISTED_COLUMNS { + format!(", and {} more", columns.len() - LISTED_COLUMNS) + } else { + String::new() + }; + let outputs = if columns.is_empty() { + String::new() + } else { + format!(" (it outputs {}{more})", listed.join(", ")) + }; + Ok(Some(format!( + "'{table}' does not output '{name}'{outputs}; update the test to its current columns" + ))) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs new file mode 100644 index 0000000000..c8b2a98b90 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs @@ -0,0 +1,190 @@ +//! Python's `_location` and the authored text a resource SQL finding quotes. + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_ignorecase_key::python_ignorecase_key; +use sqlbuild_core::text::main::python_splitlines::python_splitlines; + +use crate::semantic_checks::_helpers::resource_sql::resources::{ResourceSql, find_code_points}; +use crate::semantic_checks::_helpers::resource_sql::scanning::word_at; +use crate::semantic_checks::models::ResourceSqlFailure; +use crate::semantic_validation::models::{BindingPositions, PositionInput}; + +/// A binding finding: code, message, line, column, start, end and severity. +pub(crate) type Finding = ( + String, + String, + Option, + Option, + Option, + Option, + String, +); + +/// `(line, column, end line, end column)`. +pub(crate) type Span = (i64, i64, Option, Option); + +/// Python's `_offset_location`: the 1-based line and column of a code-point offset. +pub(crate) fn offset_location(text: &str, offset: usize) -> (i64, i64) { + let before: Vec = text.chars().take(offset).collect(); + let line = before + .iter() + .filter(|character| **character == '\n') + .count() as i64 + + 1; + let line_start = before + .iter() + .rposition(|character| *character == '\n') + .map_or(0, |index| index + 1); + (line, (offset - line_start) as i64 + 1) +} + +/// The single-quoted pieces of `message`, as `'([^']+)'` finds them. +fn quoted_names(message: &str) -> Vec<&str> { + let mut names: Vec<&str> = Vec::new(); + let mut rest = message; + while let Some(open) = rest.find('\'') { + let after = &rest[open + 1..]; + match after.find('\'') { + Some(0) => rest = after, + Some(close) => { + names.push(&after[..close]); + rest = &after[close + 1..]; + } + None => break, + } + } + names +} + +/// `(? Option { + let python = active_python_text(); + let key = |character: &char| python_ignorecase_key(python, *character); + let keys: Vec = identifier.iter().map(key).collect(); + let last = text.len().checked_sub(identifier.len())?; + (start..=last).find(|position| { + (*position == start || !word_at(text, Some(position - 1))) + && text[*position..position + identifier.len()] + .iter() + .map(key) + .eq(keys.iter().copied()) + && !word_at(text, Some(position + identifier.len())) + }) +} + +/// Python's `_quoted_name_offset`: where a name the message quotes first occurs in the body. +fn quoted_name_offset(text: &[char], start: usize, message: &str) -> Option { + quoted_names(message).into_iter().find_map(|name| { + let identifier: Vec = name.rsplit('.').next().unwrap_or(name).chars().collect(); + word_offset(text, start.min(text.len()), &identifier) + }) +} + +fn usize_line(value: Option) -> Option { + value.map(|value| value as i64) +} + +/// Python's `get_authored_binding_location` for a resource body, without expansion passes. +fn mapped_span( + resource: &ResourceSql, + cleaned_sql: &str, + finding: &Finding, +) -> Result, ResourceSqlFailure> { + let mapper = BindingPositions::new(PositionInput { + authored: resource.contents.to_string(), + query: resource.authored_body.clone(), + expanded: resource.sql.clone(), + cleaned: cleaned_sql.to_owned(), + passes: Vec::new(), + }) + .map_err(ResourceSqlFailure::Normalization)?; + let (line, column) = mapper + .position(finding.4, finding.2, finding.3, &finding.1) + .map_err(ResourceSqlFailure::Normalization)?; + let (Some(line), Some(column)) = (line, column) else { + return Ok(None); + }; + let (end_line, end_column) = match finding.5 { + Some(end) => mapper + .position(Some(end), None, None, "") + .map_err(ResourceSqlFailure::Normalization)?, + None => (None, None), + }; + Ok(Some(( + line as i64, + column as i64, + usize_line(end_line), + usize_line(end_column), + ))) +} + +/// Python's `_location`: the offending authored text, falling back to the body start. +pub(crate) fn location( + resource: &ResourceSql, + cleaned_sql: &str, + finding: &Finding, +) -> Result { + let body_offset = find_code_points(&resource.contents, &resource.authored_body).unwrap_or(0); + if finding.4.is_none() { + let text: Vec = resource.contents.chars().collect(); + if let Some(offset) = quoted_name_offset(&text, body_offset, &finding.1) { + let (line, column) = offset_location(&resource.contents, offset); + return Ok((line, column, None, None)); + } + } + if let Some(span) = mapped_span(resource, cleaned_sql, finding)? { + return Ok(span); + } + let (line, column) = offset_location(&resource.contents, body_offset); + Ok((line, column, None, None)) +} + +/// Code-point starts of each `splitlines()` line of `text`. +fn line_starts(text: &str) -> Vec { + python_splitlines(text) + .into_iter() + .map(|line| { + text[..line.as_ptr() as usize - text.as_ptr() as usize] + .chars() + .count() + }) + .collect() +} + +/// Python's `_offset`: the code-point offset of a 1-based line and column, if the line exists. +fn offset(starts: &[usize], line: i64, column: i64) -> Option { + if line < 1 { + return None; + } + starts + .get((line - 1) as usize) + .map(|start| start + (column - 1).max(0) as usize) +} + +/// Python's `_text_from`: the authored text of a span, or from its start. +pub(crate) fn text_from(resource: &ResourceSql, span: Span) -> String { + let starts = line_starts(&resource.contents); + let Some(start) = offset(&starts, span.0, span.1) else { + return String::new(); + }; + let end = match (span.2, span.3) { + (Some(line), Some(column)) => offset(&starts, line, column), + _ => None, + }; + let characters = resource.contents.chars().skip(start); + match end.filter(|end| *end > start) { + Some(end) => characters.take(end - start).collect(), + None => characters.collect(), + } +} + +/// Python's `_line_at`: the authored line of a span, or empty text. +pub(crate) fn line_at(resource: &ResourceSql, span: Span) -> String { + let lines = python_splitlines(&resource.contents); + if span.0 < 1 { + return String::new(); + } + lines + .get((span.0 - 1) as usize) + .map_or_else(String::new, |line| (*line).to_owned()) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/mod.rs new file mode 100644 index 0000000000..416b609feb --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod columns; +pub(crate) mod diagnostics; +pub(crate) mod fixtures; +pub(crate) mod help; +pub(crate) mod locations; +pub(crate) mod resources; +pub(crate) mod scanning; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/resources.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/resources.rs new file mode 100644 index 0000000000..3aab21ecb7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/resources.rs @@ -0,0 +1,235 @@ +//! Python's `_audit_resources`, `_sql_test_resources` and `_hook_resources`. + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::sync::Arc; + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::semantic_checks::_helpers::resource_sql::columns::resolve_binding_column; +use crate::semantic_checks::_helpers::resource_sql::fixtures::{ + prepared_test_sql, reference_names, +}; +use crate::semantic_checks::_helpers::resource_sql::locations::offset_location; +use crate::semantic_checks::models::{ + AuditSqlInput, HookModelInput, ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, +}; + +const COLUMN_ONLY_BUILT_IN_AUDITS: [&str; 2] = ["not_null", "unique"]; +const SQL_ANALYSIS_KEY: &str = "sql_analysis"; + +/// One authored SQL body owned by a non-model resource, Python's `_ResourceSql`. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(crate) struct ResourceSql { + pub(crate) kind: ResourceSqlKind, + pub(crate) name: String, + pub(crate) path: String, + pub(crate) contents: Arc, + pub(crate) authored_body: String, + pub(crate) sql: String, + pub(crate) relations: BTreeSet, + pub(crate) label: Option, + pub(crate) audit_definition: Option, + /// `(line, column)` of the header `sql_analysis` key that opts the resource out. + pub(crate) opt_out: Option<(i64, i64)>, +} + +fn is_blank(text: &str) -> bool { + python_strip(text).is_empty() +} + +/// `str.find` in code points, or None. +pub(crate) fn find_code_points(text: &str, needle: &str) -> Option { + text.find(needle).map(|byte| text[..byte].chars().count()) +} + +/// Python's `_header_opt_out`: where a TEST or AUDIT header turns SQL analysis off. +fn header_opt_out(opts_out: bool, contents: &str, body: &str) -> Option<(i64, i64)> { + if !opts_out { + return None; + } + let end = text_byte_end(contents, body); + let key = contents[..end] + .rfind(SQL_ANALYSIS_KEY) + .map_or(0, |byte| contents[..byte].chars().count()); + Some(offset_location(contents, key)) +} + +/// The byte end `rfind` searches before: where `body` starts, or the whole text. +fn text_byte_end(contents: &str, body: &str) -> usize { + contents.find(body).unwrap_or(contents.len()) +} + +fn audit_label(audit: &AuditSqlInput) -> String { + let mut label = format!("audit '{}'", audit.definition_name); + if let Some((kind, target)) = &audit.attached_target { + label.push_str(&format!(" on {kind} '{target}'")); + } + if let Some(column) = &audit.attached_column { + label.push_str(&format!(" column '{column}'")); + } + label +} + +fn shape<'a>(request: &'a ResourceSqlRequest, name: &str) -> &'a [(String, String)] { + request + .shapes + .iter() + .find(|(shape_name, _)| shape_name == name) + .map_or(&[], |(_, columns)| columns.as_slice()) +} + +/// Python's `_binds_cleanly`: a built-in single-column audit on a column its closed shape has. +fn binds_cleanly(audit: &AuditSqlInput, request: &ResourceSqlRequest) -> bool { + let (Some((_, target)), Some(column)) = (&audit.attached_target, &audit.attached_column) else { + return false; + }; + if !COLUMN_ONLY_BUILT_IN_AUDITS.contains(&audit.definition_name.as_str()) || !audit.built_in { + return false; + } + let columns = shape(request, target); + !columns.is_empty() + && resolve_binding_column( + column, + columns, + request.dialect.as_deref(), + request.quoted_ignore_case, + ) + .is_some() +} + +fn disabled_models(request: &ResourceSqlRequest) -> HashSet<&str> { + request + .models + .iter() + .filter(|model| model.analysis_disabled) + .map(|model| model.name.as_str()) + .collect() +} + +fn audit_resources(request: &ResourceSqlRequest, disabled: &HashSet<&str>) -> Vec { + let mut resources: Vec = Vec::new(); + for audit in &request.audits { + let target_disabled = audit + .attached_target + .as_ref() + .is_some_and(|(_, target)| disabled.contains(&target.as_str())); + if target_disabled || binds_cleanly(audit, request) { + continue; + } + for (sql, authored) in &audit.bodies { + let Some(sql) = sql.as_deref().filter(|sql| !is_blank(sql)) else { + continue; + }; + let body = authored + .as_deref() + .filter(|authored| !authored.is_empty()) + .unwrap_or(sql); + resources.push(ResourceSql { + kind: ResourceSqlKind::Audit, + name: audit.name.clone(), + path: audit.path.clone(), + contents: audit.contents.clone(), + authored_body: body.to_owned(), + sql: sql.to_owned(), + relations: audit.relations.iter().cloned().collect(), + label: Some(audit_label(audit)), + audit_definition: Some(audit.definition_name.clone()), + opt_out: header_opt_out(audit.opts_out, &audit.contents, body), + }); + } + } + resources +} + +fn test_resource(test: &SqlTestSqlInput, prepared: &(String, Vec)) -> ResourceSql { + ResourceSql { + kind: ResourceSqlKind::SqlTest, + name: test.name.clone(), + path: test.path.clone(), + contents: test.contents.clone(), + authored_body: test.authored_body.clone(), + sql: prepared.0.clone(), + relations: prepared.1.iter().cloned().collect(), + label: Some(format!("SQL test '{}'", test.name)), + audit_definition: None, + opt_out: header_opt_out(test.opts_out, &test.contents, &test.authored_body), + } +} + +fn sql_test_resources(request: &ResourceSqlRequest, disabled: &HashSet<&str>) -> Vec { + let mut prepared: HashMap<&str, (String, Vec)> = HashMap::new(); + let mut resources: Vec = Vec::new(); + for test in &request.sql_tests { + let expects_disabled = test + .expected_model_names + .iter() + .any(|name| disabled.contains(&name.as_str())); + if is_blank(&test.sql_body) || expects_disabled { + continue; + } + let sql = prepared + .entry(&test.sql_body) + .or_insert_with(|| prepared_test_sql(&test.sql_body)); + resources.push(test_resource(test, sql)); + } + resources +} + +fn hook_resources_of(model: &HookModelInput, request: &ResourceSqlRequest) -> Vec { + let mut resources: Vec = Vec::new(); + for hook in model.hooks.iter().filter(|hook| !is_blank(&hook.statement)) { + let named_file = hook.hook_path.as_ref().and_then(|path| { + request + .hook_files + .iter() + .rev() + .find(|(file, _)| file == path) + .map(|(file, contents)| (file.clone(), contents.clone())) + }); + let statement: Vec = hook.statement.chars().collect(); + let mut relations: BTreeSet = hook.relations.iter().cloned().collect(); + relations.extend(reference_names(&statement)); + let authored_body = match (&named_file, &hook.definition) { + (Some(_), Some(definition)) => definition.clone(), + _ => hook.statement.clone(), + }; + let (path, contents) = + named_file.unwrap_or_else(|| (model.path.clone(), model.authored_sql.clone())); + resources.push(ResourceSql { + kind: ResourceSqlKind::Model, + name: model.name.clone(), + path, + contents, + authored_body, + sql: hook.statement.clone(), + relations, + label: Some(format!("{}[{}] SQL hook", hook.key, hook.index)), + audit_definition: None, + opt_out: None, + }); + } + resources +} + +/// Every checked resource body, deduplicated in first-seen order and filtered by the opt-out policy. +pub(crate) fn resources(request: &ResourceSqlRequest) -> Vec { + let disabled = disabled_models(request); + let mut all = audit_resources(request, &disabled); + all.extend(sql_test_resources(request, &disabled)); + for model in request + .models + .iter() + .filter(|model| !model.analysis_disabled) + { + all.extend(hook_resources_of(model, request)); + } + let mut seen: HashSet = HashSet::with_capacity(all.len()); + let mut unique: Vec = Vec::with_capacity(all.len()); + for resource in all { + if seen.insert(resource.clone()) { + unique.push(resource); + } + } + unique.retain(|resource| resource.opt_out.is_none() || request.require_sql_analysis); + unique +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/scanning.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/scanning.rs new file mode 100644 index 0000000000..19f99d167e --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/scanning.rs @@ -0,0 +1,66 @@ +//! Python `re` building blocks over code points: `\s`, `\w`, `\b` and ignore-case words. + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::is_python_word::is_python_word; + +use crate::semantic_checks::_helpers::sql_text::python_regex::ignorecase_equal; + +/// Python's `\w` for the character at `index`, false past either end. +pub(crate) fn word_at(text: &[char], index: Option) -> bool { + index + .and_then(|index| text.get(index)) + .is_some_and(|character| is_python_word(active_python_text(), *character)) +} + +/// Python's `\b` before `index`, where the next character is a word character. +pub(crate) fn starts_word(text: &[char], index: usize) -> bool { + !word_at(text, index.checked_sub(1)) +} + +/// The end of `\s*` (or `\s+` when `required`) from `index`. +pub(crate) fn spaces(text: &[char], index: usize, required: bool) -> Option { + let end = index + + text[index.min(text.len())..] + .iter() + .take_while(|character| is_python_space(**character)) + .count(); + (!required || end > index).then_some(end) +} + +/// The end of `\w+` from `index`. +pub(crate) fn words(text: &[char], index: usize) -> Option { + let mut end = index; + while word_at(text, Some(end)) { + end += 1; + } + (end > index).then_some(end) +} + +/// The end of the ASCII `literal` at `index`, matched under `re.IGNORECASE` when `ignore_case`. +pub(crate) fn literal( + text: &[char], + index: usize, + literal: &str, + ignore_case: bool, +) -> Option { + let mut end = index; + for expected in literal.chars() { + let actual = *text.get(end)?; + let matched = if ignore_case && expected.is_ascii_alphabetic() { + ignorecase_equal(actual, expected) + } else { + actual == expected + }; + if !matched { + return None; + } + end += 1; + } + Some(end) +} + +/// The text of `text[start..end]`. +pub(crate) fn slice(text: &[char], start: usize, end: usize) -> String { + text[start..end].iter().collect() +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/check_semantic_metadata.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/check_semantic_metadata.rs index 94a9821949..206acec044 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/main/check_semantic_metadata.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/check_semantic_metadata.rs @@ -13,6 +13,7 @@ pub fn check_semantic_metadata( ) -> Result { let pool = catalog.analysis_pool().map_err(SemanticFailure::Internal)?; pool.install(|| { - catch_compiler_panic(|| Ok(check_metadata(request))).map_err(SemanticFailure::Internal)? + catch_compiler_panic(|| Ok(check_metadata(request, catalog))) + .map_err(SemanticFailure::Internal)? }) } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs index 9dd5e1a128..95cdc4c65b 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs @@ -2,3 +2,4 @@ pub mod check_semantic_metadata; pub mod complete_semantic_diagnostics; pub mod finish_type_recovery; pub mod plan_type_recovery; +pub mod resource_sql_diagnostics; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/resource_sql_diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/resource_sql_diagnostics.rs new file mode 100644 index 0000000000..f53ae9633c --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/resource_sql_diagnostics.rs @@ -0,0 +1,18 @@ +//! Python's `get_resource_sql_diagnostics`: audit, SQL test and SQL hook findings. + +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::semantic_checks::_helpers::resource_sql::diagnostics::resource_sql_findings; +use crate::semantic_checks::models::{ + ResourceSqlDiagnostic, ResourceSqlFailure, ResourceSqlRequest, +}; +use crate::semantic_validation::models::ProjectCatalog; + +/// Bind every audit, SQL test and SQL hook body against the closed shapes on `catalog`. +pub fn resource_sql_diagnostics( + request: &ResourceSqlRequest, + catalog: &ProjectCatalog, +) -> Result, ResourceSqlFailure> { + catch_compiler_panic(|| Ok(resource_sql_findings(request, catalog))) + .map_err(ResourceSqlFailure::Internal)? +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/models.rs b/crates/sqlbuild-analysis/src/semantic_checks/models.rs index 3ee67376e5..d12faf7325 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/models.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/models.rs @@ -267,6 +267,46 @@ pub struct MetadataModel { pub cursor_inputs: Vec<(String, Vec)>, pub cursor: Option, pub cursor_type: Option, + /// The model's schema audits, model-level ones first, in Python's order. + pub audits: Vec, +} + +/// One accepted value of an `accepted_values` audit, by Python type. +#[derive(Clone, Debug, PartialEq)] +pub enum AuditValue { + Null, + Bool(bool), + /// An `int` or `float`, as `str(value)`. + Number(String), + Text(String), + /// Any other object, which keeps the audit from being checked. + Other, +} + +/// One schema audit as the metadata checks read it. +#[derive(Clone, Debug, PartialEq)] +pub struct MetadataAudit { + /// The schema column the audit is attached to, if any. + pub column: Option, + pub definition: String, + /// The `expression` argument, when it is a string. + pub expression: Option, + /// `str(arguments.get("to", ""))`. + pub target: String, + /// `str(arguments.get("field", ""))`. + pub field: String, + /// The `values` argument, when it is a list or tuple. + pub values: Option>, +} + +/// One schema audit diagnostic, located at its name in the model's authored SQL. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuditFinding { + pub code: String, + pub message: String, + pub line: i64, + pub column: i64, + pub is_error: bool, } /// One source's cursor column and file text, shared with the other sources of its file. @@ -285,10 +325,12 @@ pub struct MetadataSqlTest { pub ctes: Vec<(String, Vec)>, } -/// Python's `get_semantic_metadata_diagnostics` inputs, apart from audits and resource SQL. +/// Python's `get_semantic_metadata_diagnostics` inputs, apart from resource SQL. #[derive(Clone, Debug)] pub struct MetadataRequest { pub dialect: Option, + /// The inference profile's `quoted_identifiers_ignore_case`. + pub quoted_ignore_case: bool, /// The inference profile's declared function return types. pub return_types: Vec<(String, String)>, pub functions: Vec, @@ -312,6 +354,8 @@ pub struct MetadataFinding { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ModelMetadataFindings { pub functions: Vec, + /// Schema audit findings, which follow function errors and precede reference errors. + pub audits: Vec, pub references: Vec, } @@ -325,3 +369,104 @@ pub struct MetadataOutcome { pub sql_tests: Vec<(usize, MetadataFinding, i64)>, pub fallback_types: Vec, } + +/// The kind of resource owning checked SQL, by Python's `CompiledResourceType` value. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ResourceSqlKind { + Audit, + SqlTest, + Model, +} + +/// One compiled audit as the resource SQL checks read it. +#[derive(Clone, Debug)] +pub struct AuditSqlInput { + pub name: String, + pub definition_name: String, + /// `(target kind, target name)` the audit is attached to. + pub attached_target: Option<(String, String)>, + pub attached_column: Option, + /// Whether the audit file lives under the `` root. + pub built_in: bool, + pub path: String, + pub contents: Arc, + /// `(compiled SQL, authored SQL)` of the body, measure and evidence queries. + pub bodies: Vec<(Option, Option)>, + /// Whether the AUDIT header sets `sql_analysis false`. + pub opts_out: bool, + pub relations: Vec, +} + +/// One compiled SQL test as the resource SQL checks read it. +#[derive(Clone, Debug)] +pub struct SqlTestSqlInput { + pub name: String, + pub expected_model_names: Vec, + pub path: String, + pub contents: Arc, + pub sql_body: String, + pub authored_body: String, + pub opts_out: bool, +} + +/// One SQL hook of a model: its config key, index, statement, named file and definition. +#[derive(Clone, Debug)] +pub struct HookSqlInput { + pub key: String, + pub index: usize, + pub statement: String, + pub hook_path: Option, + pub definition: Option, + pub relations: Vec, +} + +/// One model's SQL hooks as the resource SQL checks read them. +#[derive(Clone, Debug)] +pub struct HookModelInput { + pub name: String, + pub path: String, + pub authored_sql: Arc, + pub analysis_disabled: bool, + pub hooks: Vec, +} + +/// Python's `get_resource_sql_diagnostics` inputs as plain data. +#[derive(Clone, Debug)] +pub struct ResourceSqlRequest { + pub dialect: Option, + pub quoted_ignore_case: bool, + pub require_sql_analysis: bool, + /// Closed relation shapes in Python's dict order. + pub shapes: Vec<(String, Vec<(String, String)>)>, + pub audits: Vec, + pub sql_tests: Vec, + pub models: Vec, + /// `(relative path, contents)` of each SQL hook file. + pub hook_files: Vec<(String, Arc)>, +} + +/// A located resource SQL finding, or a rejected `sql_analysis false` with its note. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceSqlDiagnostic { + pub kind: ResourceSqlKind, + pub resource_name: String, + pub is_error: bool, + pub code: String, + pub message: String, + pub path: String, + /// `(line, column, end line, end column)`. + pub span: (i64, i64, Option, Option), + pub note: Option, + pub help: Option, +} + +/// Why the resource SQL checks stop: Python's error for the same input, or an internal failure. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ResourceSqlFailure { + /// The `CompileInputError` cursor intrinsic replacement raises. + CursorIntrinsic(String), + /// The `ValueError` analysis normalization raises. + Normalization(String), + /// An internal native failure, raised as `NativeCompilerError`. + Internal(String), +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests.rs index 39f180ba86..ee328b5a14 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests.rs @@ -16,6 +16,8 @@ mod operand_types; mod parsed_sql; #[path = "tests/test_positions.rs"] mod positions; +#[path = "tests/test_resource_sql.rs"] +mod resource_sql; #[path = "tests/test_types.rs"] mod test_types; #[path = "tests/test_type_recovery.rs"] diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs index 705a90481b..8495d3de07 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs @@ -549,6 +549,7 @@ fn metadata_model(name: &str, query_sql: &str, checked: bool) -> MetadataModel { ], cursor: Some("label".to_owned()), cursor_type: Some("timestamp".to_owned()), + audits: Vec::new(), } } @@ -559,6 +560,7 @@ pub(crate) fn metadata_request(test_case: &MetadataTestCase) -> MetadataRequest __udf(\"label_status\")(o.status, 1) AS label\nFROM __ref(\"stg\") AS o"; MetadataRequest { dialect: Some(test_case.dialect.to_owned()), + quoted_ignore_case: false, return_types: Vec::new(), functions: vec![ MetadataFunction { @@ -654,6 +656,7 @@ pub(crate) fn expected_checked_outcome() -> MetadataOutcome { 10, ), ], + audits: Vec::new(), references: vec![ metadata_finding( "B300", diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs new file mode 100644 index 0000000000..e4ea278d64 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs @@ -0,0 +1,43 @@ +use crate::semantic_checks::_helpers::resource_sql::fixtures::prepared_test_sql; +use crate::semantic_checks::tests::test_types::PreparedTestSqlTestCase; + +#[test] +fn given_sql_test_bodies_when_preparing_then_empty_fixtures_open_as_python_rewrites_them() { + let test_cases = [ + PreparedTestSqlTestCase { + description: "an empty fixture CTE reads its own relation", + sql_body: "WITH __ref__customers AS (SELECT * FROM __empty_fixture()) SELECT 1", + expected_sql: "WITH __ref__customers AS (SELECT * FROM customers) SELECT 1", + expected_relations: &["customers"], + }, + PreparedTestSqlTestCase { + description: "other empty fixtures open and reference calls are relations", + sql_body: "SELECT * FROM __EMPTY_FIXTURE ( ) JOIN __ref('orders') AS o ON TRUE", + expected_sql: "SELECT * FROM __sqlbuild_empty_fixture JOIN __ref('orders') AS o ON TRUE", + expected_relations: &["orders"], + }, + PreparedTestSqlTestCase { + description: "a fixture name inside a longer word is not a CTE", + sql_body: "WITH x__ref__a AS (SELECT * FROM __empty_fixture()) SELECT 1", + expected_sql: "WITH x__ref__a AS (SELECT * FROM __sqlbuild_empty_fixture) SELECT 1", + expected_relations: &[], + }, + ]; + + for test_case in test_cases { + let (sql, relations) = prepared_test_sql(test_case.sql_body); + + assert_eq!( + ( + sql.as_str(), + relations.iter().map(String::as_str).collect::>() + ), + ( + test_case.expected_sql, + test_case.expected_relations.to_vec() + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs index 9300442f68..e765fe752b 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs @@ -94,3 +94,10 @@ pub(crate) struct MetadataTestCase { pub(crate) mart_shape: &'static [(&'static str, &'static str)], pub(crate) expected_outcome: Result, } + +pub(crate) struct PreparedTestSqlTestCase { + pub(crate) description: &'static str, + pub(crate) sql_body: &'static str, + pub(crate) expected_sql: &'static str, + pub(crate) expected_relations: &'static [&'static str], +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/attachment_checks.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/attachment_checks.rs new file mode 100644 index 0000000000..72aac87153 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/attachment_checks.rs @@ -0,0 +1,486 @@ +//! `helper_ctes`: the helper CTEs a model test reads and the references it cannot resolve. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::assembly::analysis_session::main::sql_test_helper_reads::sql_test_helper_reads; +use crate::compiler::models::SqlTestCteText; +use crate::sql_test_attachment::constants::{ + DBT_REF_KIND, MOCK_PREFIXES, REF_KIND, REF_PREFIX, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, +}; +use crate::sql_test_attachment::models::{ + ModelTestCtes, SqlTestCheckFailure, SqlTestHelperProblem, SqlTestReadFacts, +}; +use crate::sql_test_attachment::types::SqlTestTextMatcher; + +type Checked = Result>; +/// The parsed reads, `None` where the body does not parse, and the reads used. +type CteReads = (Option>, Vec); + +/// `SqlTestCteGraph`: the authored CTEs by casefolded name and what each one reads. +struct CteGraph<'a> { + ctes: HashMap, + reads: HashMap>, + parsed_reads: HashMap>, + reader_reads: Vec, +} + +impl CteGraph<'_> { + fn name(&self, key: &str) -> &str { + self.ctes.get(key).map_or("", |cte| cte.0.as_str()) + } + + fn sql(&self, key: &str) -> &str { + self.ctes.get(key).map_or("", |cte| cte.1.as_str()) + } + + /// `reachable_cte_keys`: the roots then what they read, breadth first, in visit order. + fn reachable(&self, roots: &[String]) -> Vec { + let mut pending: VecDeque = roots + .iter() + .filter(|root| self.ctes.contains_key(*root)) + .cloned() + .collect(); + let mut visited: Vec = Vec::new(); + for root in &pending { + if !visited.contains(root) { + visited.push(root.clone()); + } + } + while let Some(key) = pending.pop_front() { + for dependency in self.reads.get(&key).into_iter().flatten() { + if !visited.contains(dependency) { + visited.push(dependency.clone()); + pending.push_back(dependency.clone()); + } + } + } + visited + } + + /// `_read_helper_keys`: reachable authored CTEs that are not mocks. + fn read_helpers(&self) -> Vec { + self.reachable(&self.reader_reads) + .into_iter() + .filter(|key| !is_mock(self.name(key))) + .collect() + } +} + +fn is_mock(name: &str) -> bool { + MOCK_PREFIXES.iter().any(|prefix| name.starts_with(prefix)) +} + +fn casefold(text: &str) -> String { + python_casefold(active_python_text(), text) +} + +fn unique(items: impl IntoIterator) -> Vec { + let mut seen: HashSet = HashSet::new(); + items + .into_iter() + .filter(|item| seen.insert(item.clone())) + .collect() +} + +/// `_parsed_reads` then `_token_reads` when the body does not parse. +fn cte_reads( + sql: &str, + keys: &[String], + matcher: &M, +) -> Checked { + let mut sorted: Vec = keys.to_vec(); + sorted.sort(); + if let Some(parsed) = + sql_test_helper_reads(sql, &sorted).map_err(SqlTestCheckFailure::Internal)? + { + return Ok((Some(parsed.clone()), parsed)); + } + let tokens: Vec = matcher + .identifier_tokens(sql) + .map_err(SqlTestCheckFailure::Matcher)?; + Ok(( + None, + tokens + .into_iter() + .filter(|token| keys.contains(token)) + .collect(), + )) +} + +/// `sql_test_cte_graph` over the test's authored CTEs and its expected and assertion readers. +fn cte_graph<'a, M: SqlTestTextMatcher>( + payload: &ModelTestCtes<'a>, + matcher: &M, +) -> Checked, M::Error> { + let mut keys: Vec = Vec::new(); + let mut ctes: HashMap = HashMap::new(); + for cte in payload.authored { + let key: String = casefold(&cte.0); + if ctes.insert(key.clone(), cte).is_none() { + keys.push(key); + } + } + let mut reads: HashMap> = HashMap::new(); + let mut parsed_reads: HashMap> = HashMap::new(); + for key in &keys { + let others: Vec = keys.iter().filter(|other| *other != key).cloned().collect(); + let (parsed, read) = cte_reads(&ctes[key].1, &others, matcher)?; + parsed_reads.insert(key.clone(), parsed.unwrap_or_default()); + reads.insert(key.clone(), read); + } + let mut reader_reads: Vec = Vec::new(); + for reader in payload.readers() { + reader_reads.extend(cte_reads(&reader.1, &keys, matcher)?.1); + } + let _ = keys; + Ok(CteGraph { + ctes, + reads, + parsed_reads, + reader_reads: unique(reader_reads), + }) +} + +/// `scan_sql_reference_calls(...).references`, raising the scan's unlocated error. +fn references(sql: &str, syntax: &LexicalSyntax) -> Checked, E> { + match extract_sql_references(sql, syntax) { + ReferenceExtraction::Extracted(scan) => Ok(scan.references), + ReferenceExtraction::Failed(failure) => Err(SqlTestCheckFailure::Scan(failure.message)), + } +} + +/// `SqlReferenceKind.example_call(*args, quote='"')`. +fn example_call(kind: &str, arguments: &[&str]) -> String { + let quoted: Vec = arguments + .iter() + .map(|argument| format!("\"{}\"", argument.replace('"', "\"\""))) + .collect(); + format!("__{kind}({})", quoted.join(", ")) +} + +/// `_reference_call`: a dbt reference names its package first. +fn reference_call(reference: &SqlReference) -> String { + match (&reference.package, reference.kind) { + (Some(package), DBT_REF_KIND) => example_call(DBT_REF_KIND, &[package, &reference.name]), + _ => example_call(reference.kind, &[&reference.name]), + } +} + +/// `sql_test_reads`: the helpers a test reads and the unmocked models they and expected rows read. +pub(crate) fn sql_test_reads( + payload: &ModelTestCtes<'_>, + syntax: &LexicalSyntax, + matcher: &M, +) -> Checked { + let graph: CteGraph<'_> = cte_graph(payload, matcher)?; + let read_helpers: Vec = graph.read_helpers(); + let sqls = payload + .expected + .iter() + .map(|cte| cte.1.as_str()) + .chain(read_helpers.iter().map(|key| graph.sql(key))); + let mut targets: Vec = Vec::new(); + for sql in sqls { + for reference in references(sql, syntax)? { + if reference.kind == REF_KIND && !payload.mock_models.contains(&reference.name) { + targets.push(reference.name); + } + } + } + Ok(SqlTestReadFacts { + read_helper_names: read_helpers + .iter() + .map(|key| graph.name(key).to_owned()) + .collect(), + reference_target_model_names: unique(targets), + }) +} + +/// `report_unresolvable_test_references`: each unresolvable reference, then any read cycle. +pub(crate) fn unresolvable_test_references( + payload: &ModelTestCtes<'_>, + known_models: &HashSet, + syntax: &LexicalSyntax, + matcher: &M, +) -> Checked, M::Error> { + let graph: CteGraph<'_> = cte_graph(payload, matcher)?; + let read_helpers: Vec = graph.read_helpers(); + let readers = read_helpers + .iter() + .map(|key| ("helper CTE", graph.name(key), graph.sql(key))) + .chain( + payload + .expected + .iter() + .map(|cte| ("expected CTE", cte.0.as_str(), cte.1.as_str())), + ) + .chain( + payload + .assertions + .iter() + .map(|cte| ("assertion CTE", cte.0.as_str(), cte.1.as_str())), + ); + let mut problems: Vec = Vec::new(); + for (label, name, sql) in readers { + for reference in references(sql, syntax)? { + let Some(mocked) = payload.mocked(reference.kind) else { + continue; + }; + let mock_name: String = match &reference.package { + Some(package) => format!("{package}__{}", reference.name), + None => reference.name.clone(), + }; + if mocked.contains(&mock_name) { + continue; + } + if reference.kind == REF_KIND { + if known_models.contains(&reference.name) { + continue; + } + problems.push(SqlTestHelperProblem { + cte_name: name.to_owned(), + call: Some(example_call(REF_KIND, &[&reference.name])), + message: format!( + "SQL test {label} '{name}' references unknown model '{}'", + reference.name + ), + help: format!( + "Read a model defined in the project, for example __ref(\"\"), or \ + remove the reference from {label} '{name}'." + ), + }); + continue; + } + let call: String = reference_call(&reference); + problems.push(SqlTestHelperProblem { + cte_name: name.to_owned(), + call: Some(call.clone()), + message: format!( + "SQL test {label} '{name}' calls {call}, which the test does not mock, so \ + the test query cannot resolve it" + ), + help: format!( + "Mock it in the test, for example __{}__{mock_name} AS (SELECT ...), or \ + remove the call from {label} '{name}'.", + reference.kind + ), + }); + } + } + let cycle: Option> = if read_helpers.is_empty() { + None + } else { + first_cycle(&graph, &graph.reachable(&graph.reader_reads)) + }; + if let Some(cycle) = cycle { + let names: Vec<&str> = cycle.iter().map(|key| graph.name(key)).collect(); + let path: String = names + .iter() + .map(|name| format!("'{name}'")) + .collect::>() + .join(" -> "); + problems.push(SqlTestHelperProblem { + cte_name: names[0].to_owned(), + call: None, + message: format!( + "SQL test CTE '{}' reads itself through {path}, so the test query cannot define \ + its CTEs in dependency order", + names[0] + ), + help: "Break the cycle by moving the shared rows into a helper CTE that reads \ + neither, for example shared_rows AS (SELECT ...), and read shared_rows from \ + both CTEs." + .to_owned(), + }); + } + Ok(problems) +} + +/// `_first_cycle`: the first parsed read cycle among `keys`, closed on its first CTE. +fn first_cycle(graph: &CteGraph<'_>, keys: &[String]) -> Option> { + let allowed: HashSet<&String> = keys.iter().collect(); + for origin in keys { + let mut pending: VecDeque> = VecDeque::from([vec![origin.clone()]]); + let mut visited: HashSet = HashSet::new(); + while let Some(path) = pending.pop_front() { + let last: &String = path.last()?; + for dependency in graph.parsed_reads.get(last).into_iter().flatten() { + if dependency == origin { + let mut closed: Vec = path.clone(); + closed.push(origin.clone()); + return Some(closed); + } + if allowed.contains(dependency) && visited.insert(dependency.clone()) { + let mut next: Vec = path.clone(); + next.push(dependency.clone()); + pending.push_back(next); + } + } + } + } + None +} + +/// `report_test_without_target_model`: the problem of a model test whose checks read only mocks. +pub(crate) fn test_without_target_model( + payload: &ModelTestCtes<'_>, + test_name: &str, + syntax: &LexicalSyntax, + matcher: &M, +) -> Checked { + let graph: CteGraph<'_> = cte_graph(payload, matcher)?; + let mut read_mocks: Vec = graph + .reachable(&graph.reader_reads) + .iter() + .map(|key| graph.name(key).to_owned()) + .filter(|name| is_mock(name)) + .collect(); + let mut first_calls: Vec<(&'static str, (String, String))> = Vec::new(); + let mut called_models: HashSet = HashSet::new(); + for reader in payload.readers() { + for reference in references(&reader.1, syntax)? { + let mocked: bool = payload + .mocked(reference.kind) + .or_else(|| { + (reference.kind == TABLE_FUNCTION_KIND).then_some(payload.mock_table_functions) + }) + .is_some_and(|names| names.contains(&reference.name)); + if !mocked { + continue; + } + read_mocks.push(format!("__{}__{}", reference.kind, reference.name)); + if reference.kind == REF_KIND { + called_models.insert(reference.name.clone()); + } + if reference.kind != TABLE_FUNCTION_KIND + && !first_calls.iter().any(|(kind, _)| *kind == reference.kind) + { + first_calls.push(( + reference.kind, + (reader.0.clone(), reference_call(&reference)), + )); + } + } + } + let mocks: Vec = unique(read_mocks); + let mocked_models: Vec<&str> = mocks + .iter() + .filter_map(|mock| mock.strip_prefix(REF_PREFIX)) + .collect(); + let located: Option<(String, String)> = + match first_calls.iter().find(|(kind, _)| *kind == REF_KIND) { + Some((_, call)) => Some(call.clone()), + None => match mock_name_read(payload, &mocks, matcher)? { + Some(read) => Some(read), + None => first_calls.first().map(|(_, call)| call.clone()), + }, + }; + let (message, help) = if mocked_models.is_empty() { + let reads: String = if mocks.is_empty() { + "reads no model".to_owned() + } else { + format!("reads only mocks ({})", mocks.join(", ")) + }; + ( + format!("SQL test '{test_name}' {reads}, so the test has no model to run against"), + "Call __ref(\"\") in an __assert__ or __expected__ CTE so the test \ + runs that model with its mocks, or remove the test." + .to_owned(), + ) + } else { + let plural: &str = if mocked_models.len() > 1 { "s" } else { "" }; + let owner: &str = if plural.is_empty() { + "model's" + } else { + "models'" + }; + let verb: &str = if mocked_models + .iter() + .all(|name| called_models.contains(*name)) + { + "keep" + } else { + "call" + }; + let mocked_ctes: String = mocked_models + .iter() + .map(|name| format!("{REF_PREFIX}{name}")) + .collect::>() + .join(", "); + let calls: String = mocked_models + .iter() + .map(|name| example_call(REF_KIND, &[name])) + .collect::>() + .join(" and "); + ( + format!( + "SQL test '{test_name}' mocks the model{plural} it tests ({mocked_ctes}), so the \ + test has no model to run against" + ), + format!( + "Mock the {owner} inputs instead (for example __source__ or \ + __ref__) and {verb} {calls} in the __assert__ or __expected__ \ + CTE, or remove the test." + ), + ) + }; + let (cte_name, call) = match located { + Some((cte, call)) => (cte, Some(call)), + None => ( + payload + .readers() + .next() + .map_or_else(|| test_name.to_owned(), |reader| reader.0.clone()), + None, + ), + }; + Ok(SqlTestHelperProblem { + cte_name, + call, + message, + help, + }) +} + +/// `_mock_name_read`: the first check reading a mocked model by its CTE name. +fn mock_name_read( + payload: &ModelTestCtes<'_>, + mocks: &[String], + matcher: &M, +) -> Checked, M::Error> { + for mock in mocks.iter().filter(|mock| mock.starts_with(REF_PREFIX)) { + for reader in payload.readers() { + if matcher + .reads_word(&reader.1, mock) + .map_err(SqlTestCheckFailure::Matcher)? + { + return Ok(Some((reader.0.clone(), mock.clone()))); + } + } + } + Ok(None) +} + +impl<'a> ModelTestCtes<'a> { + /// `_reader_ctes`: the expected, then assertion, CTEs. + pub(crate) fn readers(&self) -> impl Iterator { + self.expected.iter().chain(self.assertions.iter()) + } + + /// The mocked names of a `ref`, `source`, `seed` or `dbt_ref` reference kind. + fn mocked(&self, kind: &str) -> Option<&'a [String]> { + match kind { + REF_KIND => Some(self.mock_models), + SOURCE_KIND => Some(self.mock_sources), + SEED_KIND => Some(self.mock_seeds), + DBT_REF_KIND => Some(self.mock_dbt_refs), + _ => None, + } + } +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/authored_offsets.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/authored_offsets.rs new file mode 100644 index 0000000000..132322797a --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/authored_offsets.rs @@ -0,0 +1,155 @@ +//! `authored_file_offset`: an offset in a cleaned SQL-test body mapped back to the file. + +use sqlbuild_core::text::main::is_python_space::is_python_space; + +use crate::sql_test_attachment::constants::TAB_SIZE; + +/// One kept line of the cleaned body: its file offset, raw text and removed indent columns. +struct CleanedLine { + start: usize, + text: Vec, + removed: usize, +} + +/// `str.expandtabs()` in code points; line breaks reset the column. +fn expand_tabs(text: &[char]) -> Vec { + let mut expanded: Vec = Vec::with_capacity(text.len()); + let mut column: usize = 0; + for &character in text { + match character { + '\t' => { + let width: usize = TAB_SIZE - column % TAB_SIZE; + expanded.extend(std::iter::repeat_n(' ', width)); + column += width; + } + '\n' | '\r' => { + expanded.push(character); + column = 0; + } + _ => { + expanded.push(character); + column += 1; + } + } + } + expanded +} + +/// Whether `line.strip()` is non-empty. +fn has_content(line: &[char]) -> bool { + line.iter().copied().any(is_not_space) +} + +fn is_not_space(character: char) -> bool { + !is_python_space(character) +} + +fn indent(line: &[char]) -> usize { + line.iter() + .take_while(|character| is_python_space(**character)) + .count() +} + +fn tail(line: &[char], removed: usize) -> &[char] { + line.get(removed..).unwrap_or(&[]) +} + +/// `_cleaned_lines`: the kept lines when cleaning the span reproduces `authored`, else `None`. +fn cleaned_lines( + contents: &[char], + authored: &str, + span: (usize, usize), +) -> Option> { + let text: &[char] = contents.get(span.0..span.1)?; + let raw: Vec<&[char]> = text.split(|character| *character == '\n').collect(); + let expanded: Vec> = raw.iter().map(|line| expand_tabs(line)).collect(); + let margin: Option = expanded[1..] + .iter() + .filter(|line| has_content(line)) + .map(|line| indent(line)) + .min(); + let mut starts: Vec = Vec::with_capacity(raw.len()); + let mut cursor: usize = span.0; + for line in &raw { + starts.push(cursor); + cursor += line.len() + 1; + } + let mut removed: Vec = vec![indent(&expanded[0])]; + removed.extend(std::iter::repeat_n(margin.unwrap_or(0), expanded.len() - 1)); + let mut kept: Vec = (0..expanded.len()).collect(); + while kept + .last() + .is_some_and(|&index| tail(&expanded[index], removed[index]).is_empty()) + { + kept.pop(); + } + while kept + .first() + .is_some_and(|&index| tail(&expanded[index], removed[index]).is_empty()) + { + kept.remove(0); + } + let cleaned: String = kept + .iter() + .map(|&index| { + tail(&expanded[index], removed[index]) + .iter() + .collect::() + }) + .collect::>() + .join("\n"); + if cleaned != authored { + return None; + } + Some( + kept.iter() + .map(|&index| CleanedLine { + start: starts[index], + text: raw[index].to_vec(), + removed: removed[index], + }) + .collect(), + ) +} + +/// `_authored_index`: the raw index under an expanded column. +fn authored_index(text: &[char], expanded_column: usize) -> usize { + let mut position: usize = 0; + let mut column: usize = 0; + for (index, &character) in text.iter().enumerate() { + let width: usize = if character == '\t' { + TAB_SIZE - column % TAB_SIZE + } else { + 1 + }; + if expanded_column < position + width { + return index; + } + position += width; + column = if character == '\r' { 0 } else { column + width }; + } + text.len() +} + +/// The file code-point offset of `body_offset` in the cleaned `authored` body, if exact. +pub(crate) fn authored_offset( + contents: &str, + authored: &str, + span: Option<(usize, usize)>, + body_offset: usize, +) -> Option { + let contents: Vec = contents.chars().collect(); + let lines: Vec = cleaned_lines(&contents, authored, span?)?; + let body: Vec = authored.chars().collect(); + let before: &[char] = body.get(..body_offset)?; + let line_index: usize = before + .iter() + .filter(|character| **character == '\n') + .count(); + let line_start: usize = before + .iter() + .rposition(|character| *character == '\n') + .map_or(0, |index| index + 1); + let line: &CleanedLine = lines.get(line_index)?; + Some(line.start + authored_index(&line.text, line.removed + body_offset - line_start)) +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/mod.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/mod.rs new file mode 100644 index 0000000000..992760926e --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod attachment_checks; +pub(crate) mod authored_offsets; +pub(crate) mod text_matching; diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/text_matching.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/text_matching.rs new file mode 100644 index 0000000000..77cd79a7a6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/_helpers/text_matching.rs @@ -0,0 +1,130 @@ +//! Python `re` semantics of the SQL-test text checks: `\w`, `\s` and `re.IGNORECASE` literals. + +use std::collections::HashSet; + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::is_python_word::is_python_word; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_core::text::main::python_ignorecase_key::python_ignorecase_key; +use sqlbuild_core::text::models::PythonText; + +const DOLLAR: char = '$'; +const QUOTE_CHARACTERS: [char; 2] = ['"', '`']; +const AS_KEYWORD: [char; 2] = ['a', 's']; +const OPEN_PAREN: char = '('; + +/// One character of `[\w$]`. +fn identifier_character(python: PythonText, character: char) -> bool { + character == DOLLAR || is_python_word(python, character) +} + +/// The start of `[A-Za-z_]`. +fn identifier_start(character: char) -> bool { + character.is_ascii_alphabetic() || character == '_' +} + +/// `re.findall(r"[A-Za-z_][\w$]*", sql)` casefolded, each distinct token once, first first. +pub(crate) fn identifier_tokens(sql: &str) -> Vec { + let python: PythonText = active_python_text(); + let characters: Vec = sql.chars().collect(); + let mut seen: HashSet = HashSet::new(); + let mut tokens: Vec = Vec::new(); + let mut index: usize = 0; + while index < characters.len() { + if !identifier_start(characters[index]) { + index += 1; + continue; + } + let start: usize = index; + index += 1; + while index < characters.len() && identifier_character(python, characters[index]) { + index += 1; + } + let token: String = characters[start..index].iter().collect(); + let folded: String = python_casefold(python, &token); + if seen.insert(folded.clone()) { + tokens.push(folded); + } + } + tokens +} + +/// Whether `text[at..]` starts with `literal` under `re.IGNORECASE`. +fn ignorecase_at(python: PythonText, text: &[char], at: usize, literal: &[char]) -> bool { + text.len() >= at + literal.len() + && text[at..at + literal.len()] + .iter() + .zip(literal) + .all(|(left, right)| { + python_ignorecase_key(python, *left) == python_ignorecase_key(python, *right) + }) +} + +/// Whether no `[\w$]` character precedes `at`. +fn starts_word(python: PythonText, text: &[char], at: usize) -> bool { + at == 0 || !identifier_character(python, text[at - 1]) +} + +/// The first `at >= from` where `(? Option { + (from..=text.len().saturating_sub(literal.len())) + .find(|at| starts_word(python, text, *at) && ignorecase_at(python, text, *at, literal)) +} + +/// `re.search(rf"(? bool { + let python: PythonText = active_python_text(); + let text: Vec = sql.chars().collect(); + let literal: Vec = word.chars().collect(); + (0..=text.len().saturating_sub(literal.len())).any(|at| { + starts_word(python, &text, at) + && ignorecase_at(python, &text, at, &literal) + && text + .get(at + literal.len()) + .is_none_or(|next| !identifier_character(python, *next)) + }) +} + +/// After a CTE name: an optional quote, `\s+AS\s*\(`; the index after `(`, if it matches. +fn cte_header_end(python: PythonText, text: &[char], at: usize) -> Option { + let mut index: usize = at; + if text + .get(index) + .is_some_and(|character| QUOTE_CHARACTERS.contains(character)) + { + index += 1; + } + let spaces: usize = index; + while index < text.len() && is_python_space(text[index]) { + index += 1; + } + if index == spaces || !ignorecase_at(python, text, index, &AS_KEYWORD) { + return None; + } + index += AS_KEYWORD.len(); + while index < text.len() && is_python_space(text[index]) { + index += 1; + } + (text.get(index) == Some(&OPEN_PAREN)).then_some(index + 1) +} + +/// `(? Option<(usize, usize)> { + let python: PythonText = active_python_text(); + let mut search: usize = from; + while let Some(start) = word_start(python, text, search, name) { + if let Some(end) = cte_header_end(python, text, start + name.len()) { + return Some((start, end)); + } + search = start + 1; + } + None +} + +/// `re.search(re.escape(call), text, re.IGNORECASE)` from `from`. +pub(crate) fn literal_from(text: &[char], from: usize, literal: &[char]) -> Option { + let python: PythonText = active_python_text(); + (from..=text.len().saturating_sub(literal.len())) + .find(|at| ignorecase_at(python, text, *at, literal)) +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/constants.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/constants.rs new file mode 100644 index 0000000000..0f500fee6d --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/constants.rs @@ -0,0 +1,18 @@ +//! Fixed names the SQL-test attachment checks read. + +/// The mode of a stored model SQL test classification. +pub(crate) const STORED_MODEL_MODE: &str = "model"; +pub(crate) const REF_KIND: &str = "ref"; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const SEED_KIND: &str = "seed"; +pub(crate) const DBT_REF_KIND: &str = "dbt_ref"; +pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn"; +pub(crate) const REF_PREFIX: &str = "__ref__"; +pub(crate) const MOCK_PREFIXES: [&str; 5] = [ + "__ref__", + "__source__", + "__seed__", + "__dbt_ref__", + "__table_fn__", +]; +pub(crate) const TAB_SIZE: usize = 8; diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/main/authored_test_offset.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/main/authored_test_offset.rs new file mode 100644 index 0000000000..4ae393997d --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/main/authored_test_offset.rs @@ -0,0 +1,14 @@ +//! A cleaned SQL-test or scenario body offset mapped to its authored file offset. + +use crate::sql_test_attachment::_helpers::authored_offsets::authored_offset; + +/// `authored_file_offset`: the file code-point offset of `body_offset`, when it is exact. +#[must_use] +pub fn authored_test_offset( + contents: &str, + authored: &str, + span: Option<(usize, usize)>, + body_offset: usize, +) -> Option { + authored_offset(contents, authored, span, body_offset) +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/main/mod.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/main/mod.rs new file mode 100644 index 0000000000..1f668da82e --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/main/mod.rs @@ -0,0 +1,6 @@ +pub mod authored_test_offset; +pub mod sql_test_cte_location; +pub mod sql_test_reads; +pub mod stored_sql_tests; +pub mod test_without_target_model; +pub mod unresolvable_test_references; diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/main/sql_test_cte_location.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/main/sql_test_cte_location.rs new file mode 100644 index 0000000000..8a55e35ac1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/main/sql_test_cte_location.rs @@ -0,0 +1,56 @@ +//! Where a SQL-test helper problem is reported: its call, else its CTE header, else the block. + +use crate::sql_test_attachment::_helpers::text_matching::{cte_header, literal_from}; +use crate::sql_test_attachment::models::CteLocation; + +/// The code-point index of `needle` in `haystack`, as `str.find` returns it. +fn char_find(haystack: &str, needle: &str) -> Option { + haystack + .find(needle) + .map(|byte| haystack[..byte].chars().count()) +} + +/// Line and column of a code-point offset, as `reference_call_location` counts them. +fn line_column(text: &[char], offset: usize) -> (usize, usize) { + let before: &[char] = &text[..offset.min(text.len())]; + let line: usize = before + .iter() + .filter(|character| **character == '\n') + .count() + + 1; + let line_start: usize = before + .iter() + .rposition(|character| *character == '\n') + .map_or(0, |newline| newline + 1); + (line, offset - line_start + 1) +} + +/// `sql_test_cte_location`: the span of `call` inside the CTE, the CTE name, or the block start. +#[must_use] +pub fn sql_test_cte_location( + contents: &str, + sql_body: &str, + cte_name: &str, + call: Option<&str>, +) -> CteLocation { + let text: Vec = contents.chars().collect(); + let name: Vec = cte_name.chars().collect(); + let block_offset: usize = char_find(contents, sql_body).unwrap_or(0); + let header: Option<(usize, usize)> = cte_header(&text, block_offset, &name); + let (mut start, mut length) = + header.map_or((block_offset, 0), |(start, _)| (start, name.len())); + if let (Some(call), Some((_, header_end))) = (call, header) { + let literal: Vec = call.chars().collect(); + if let Some(found) = literal_from(&text, header_end, &literal) { + (start, length) = (found, literal.len()); + } + } + let (line, column) = line_column(&text, start); + let (end_line, end_column) = line_column(&text, start + length); + CteLocation { + line, + column, + end_line, + end_column, + } +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/main/sql_test_reads.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/main/sql_test_reads.rs new file mode 100644 index 0000000000..a9892707ea --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/main/sql_test_reads.rs @@ -0,0 +1,16 @@ +//! The `sql_test_reads` helper check of the compile attachment. + +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::sql_test_attachment::_helpers::attachment_checks; +use crate::sql_test_attachment::models::{ModelTestCtes, SqlTestCheckFailure, SqlTestReadFacts}; +use crate::sql_test_attachment::types::SqlTestTextMatcher; + +/// `sql_test_reads`: the helpers a test reads and the unmocked models they read. +pub fn sql_test_reads( + payload: &ModelTestCtes<'_>, + syntax: &LexicalSyntax, + matcher: &M, +) -> Result> { + attachment_checks::sql_test_reads(payload, syntax, matcher) +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/main/stored_sql_tests.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/main/stored_sql_tests.rs new file mode 100644 index 0000000000..b57ff1db55 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/main/stored_sql_tests.rs @@ -0,0 +1,22 @@ +//! One test file's stored SQL-test classifications, read back from the shared scan store. + +use crate::compiler::models::SqlTestClassification; +use crate::sql_test_attachment::constants::STORED_MODEL_MODE; + +/// The stored classifications when they decode to exactly the file's tests' kinds and modes. +#[must_use] +pub fn stored_sql_tests(value: &[u8], modes: &[&str]) -> Option> { + let stored: Vec = match serde_json::from_slice(value) { + Ok(stored) => stored, + Err(_) => return None, + }; + let matches: bool = stored.len() == modes.len() + && stored.iter().zip(modes).all(|(item, mode)| match item { + SqlTestClassification::Model { mode: stored, .. } => { + stored == STORED_MODEL_MODE && stored == mode + } + SqlTestClassification::Direct { mode: stored, .. } => stored == mode, + SqlTestClassification::Authored { .. } => false, + }); + matches.then_some(stored) +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/main/test_without_target_model.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/main/test_without_target_model.rs new file mode 100644 index 0000000000..a9bce84f88 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/main/test_without_target_model.rs @@ -0,0 +1,19 @@ +//! The `test_without_target_model` helper check of the compile attachment. + +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::sql_test_attachment::_helpers::attachment_checks; +use crate::sql_test_attachment::models::{ + ModelTestCtes, SqlTestCheckFailure, SqlTestHelperProblem, +}; +use crate::sql_test_attachment::types::SqlTestTextMatcher; + +/// `report_test_without_target_model`: the P013 problem of a test that runs no model. +pub fn test_without_target_model( + payload: &ModelTestCtes<'_>, + test_name: &str, + syntax: &LexicalSyntax, + matcher: &M, +) -> Result> { + attachment_checks::test_without_target_model(payload, test_name, syntax, matcher) +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/main/unresolvable_test_references.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/main/unresolvable_test_references.rs new file mode 100644 index 0000000000..484dd004fe --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/main/unresolvable_test_references.rs @@ -0,0 +1,21 @@ +//! The `unresolvable_test_references` helper check of the compile attachment. + +use std::collections::HashSet; + +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::sql_test_attachment::_helpers::attachment_checks; +use crate::sql_test_attachment::models::{ + ModelTestCtes, SqlTestCheckFailure, SqlTestHelperProblem, +}; +use crate::sql_test_attachment::types::SqlTestTextMatcher; + +/// `report_unresolvable_test_references`: each P013 problem, in Python's report order. +pub fn unresolvable_test_references( + payload: &ModelTestCtes<'_>, + known_models: &HashSet, + syntax: &LexicalSyntax, + matcher: &M, +) -> Result, SqlTestCheckFailure> { + attachment_checks::unresolvable_test_references(payload, known_models, syntax, matcher) +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/mod.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/mod.rs new file mode 100644 index 0000000000..34e08adebe --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/mod.rs @@ -0,0 +1,10 @@ +//! The compile attachment's checks of SQL tests: helper reads, located offsets and stored scans. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/models.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/models.rs new file mode 100644 index 0000000000..d340c36229 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/models.rs @@ -0,0 +1,67 @@ +//! The facts and problems the SQL-test attachment checks report. + +use crate::compiler::models::SqlTestCteText; +use crate::sql_test_attachment::_helpers::text_matching::{identifier_tokens, reads_word}; +use crate::sql_test_attachment::types::SqlTestTextMatcher; + +/// A model test's classified CTEs, as the attachment's helper checks read them. +#[derive(Debug, Clone, Copy)] +pub struct ModelTestCtes<'a> { + pub authored: &'a [SqlTestCteText], + pub expected: &'a [SqlTestCteText], + pub assertions: &'a [SqlTestCteText], + pub mock_models: &'a [String], + pub mock_sources: &'a [String], + pub mock_seeds: &'a [String], + pub mock_dbt_refs: &'a [String], + pub mock_table_functions: &'a [String], +} + +/// `SqlTestReads`: the helpers a test reads and the unmocked models they read. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SqlTestReadFacts { + pub read_helper_names: Vec, + pub reference_target_model_names: Vec, +} + +/// One P013 problem in a model test, located at `call` inside CTE `cte_name` when given. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SqlTestHelperProblem { + pub cte_name: String, + pub call: Option, + pub message: String, + pub help: String, +} + +/// Why a helper check stopped: an internal failure, a reference scan error, or the matcher's. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SqlTestCheckFailure { + Internal(String), + Scan(String), + Matcher(E), +} + +/// A reported span: 1-based start and end lines and columns. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CteLocation { + pub line: usize, + pub column: usize, + pub end_line: usize, + pub end_column: usize, +} + +/// Python `re` semantics of the SQL-test text checks for the running Python's Unicode tables. +#[derive(Clone, Copy, Debug, Default)] +pub struct PythonTextMatcher; + +impl SqlTestTextMatcher for PythonTextMatcher { + type Error = std::convert::Infallible; + + fn identifier_tokens(&self, sql: &str) -> Result, Self::Error> { + Ok(identifier_tokens(sql)) + } + + fn reads_word(&self, sql: &str, word: &str) -> Result { + Ok(reads_word(sql, word)) + } +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/tests.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/tests.rs new file mode 100644 index 0000000000..97fca4563a --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/tests.rs @@ -0,0 +1,12 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_sql_scenario_extraction.rs"] +mod sql_scenario_extraction; +#[path = "tests/test_sql_test_attachment_checks.rs"] +mod sql_test_attachment_checks; +#[path = "tests/test_sql_test_classification_cases.rs"] +mod sql_test_classification_cases; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_text_matching.rs"] +mod text_matching; diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/tests/helpers.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/helpers.rs new file mode 100644 index 0000000000..9a5cd87c2d --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/helpers.rs @@ -0,0 +1,154 @@ +use serde_json::{Value, json}; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::compiler::main::sql_scenario_classification::extract_scenario; +use crate::compiler::main::sql_test_extraction::extract_batch_json; +use crate::compiler::models::SqlTestCteText; +use crate::sql_test_attachment::models::{ModelTestCtes, SqlTestHelperProblem}; +use crate::sql_test_attachment::tests::test_types::AsciiMatcher; +use crate::sql_test_attachment::types::SqlTestTextMatcher; + +/// A scenario's classification as `{"scenario": ...}`, or its error as `{"error": ...}`. +pub(crate) fn scenario_json(sql: &str, file_label: &str, syntax: &LexicalSyntax) -> String { + extract_scenario(sql, file_label, syntax) + .map_or_else( + |error| json!({"error": error}), + |scenario| json!({"scenario": scenario}), + ) + .to_string() +} + +/// The batch response of one extraction request, as JSON. +pub(crate) fn batch_outcome(request: &str) -> String { + extract_batch_json(request).expect("the request is valid") +} + +/// One generic-SQL test's payload before (`raw`) or after expansion, and its P012 flag. +pub(crate) fn parity_payload(sql: &str, mode: &str, raw: bool) -> (Value, bool) { + let request: String = json!({"tests": [{"sql": sql, "fileLabel": "tests/unit/example.sql", "mode": mode, "raw": raw}]}) + .to_string(); + let mut response: Value = + serde_json::from_str(&batch_outcome(&request)).expect("the response is JSON"); + let invalid_calls: bool = response["tests"][0] + .as_object_mut() + .and_then(|test| test.remove("invalidCalls")) + .and_then(|flag| flag.as_bool()) + .unwrap_or(false); + (response, invalid_calls) +} + +/// A scenario's outcome under the lexical rules given as JSON. +pub(crate) fn scenario_outcome(sql: &str, file_label: &str, syntax: &str) -> String { + let syntax: LexicalSyntax = serde_json::from_str(syntax).expect("the syntax is JSON"); + scenario_json(sql, file_label, &syntax) +} + +/// The unique `__ref` targets the assertion bodies read, under the lexical rules given as JSON. +pub(crate) fn assertion_targets(assertions: &[&str], syntax: &str) -> Vec { + let syntax: LexicalSyntax = serde_json::from_str(syntax).expect("the syntax is JSON"); + let mut targets: Vec = Vec::new(); + for reference in assertions + .iter() + .flat_map(|sql| scanned_references(sql, &syntax)) + .filter(|reference| reference.kind == "ref") + { + targets.push(reference.name); + } + targets.dedup(); + targets +} + +fn scanned_references(sql: &str, syntax: &LexicalSyntax) -> Vec { + let ReferenceExtraction::Extracted(scan) = extract_sql_references(sql, syntax) else { + panic!("the assertion scans"); + }; + scan.references +} + +/// Generic SQL lexical rules: `--` line comments. +pub(crate) fn generic_dash_syntax() -> LexicalSyntax { + LexicalSyntax { + line_comment_prefixes: vec!["--".to_owned()], + ..LexicalSyntax::default() + } +} + +fn is_word(character: char) -> bool { + character.is_ascii_alphanumeric() || character == '_' || character == '$' +} + +fn starts_identifier(token: &&str) -> bool { + token + .chars() + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') +} + +impl SqlTestTextMatcher for AsciiMatcher { + type Error = (); + + fn identifier_tokens(&self, sql: &str) -> Result, ()> { + let mut tokens: Vec = Vec::new(); + for token in sql + .split(|character: char| !is_word(character)) + .filter(starts_identifier) + { + tokens.push(token.to_ascii_lowercase()); + } + let mut seen: Vec = Vec::new(); + tokens.retain(|token| { + let fresh: bool = !seen.contains(token); + seen.push(token.clone()); + fresh + }); + Ok(tokens) + } + + fn reads_word(&self, sql: &str, word: &str) -> Result { + Ok(sql + .to_ascii_lowercase() + .split(|character: char| !is_word(character)) + .any(|token| token == word.to_ascii_lowercase())) + } +} + +pub(crate) fn owned(items: &[&str]) -> Vec { + items.iter().map(|item| (*item).to_owned()).collect() +} + +pub(crate) fn cte_texts(ctes: &[(&str, &str)]) -> Vec { + ctes.iter() + .map(|(name, body)| SqlTestCteText((*name).to_owned(), (*body).to_owned())) + .collect() +} + +/// A model test's CTE view with only model and source mocks. +pub(crate) fn model_test_ctes<'a>( + ctes: ( + &'a [SqlTestCteText], + &'a [SqlTestCteText], + &'a [SqlTestCteText], + ), + mocks: (&'a [String], &'a [String]), +) -> ModelTestCtes<'a> { + ModelTestCtes { + authored: ctes.0, + expected: ctes.1, + assertions: ctes.2, + mock_models: mocks.0, + mock_sources: mocks.1, + mock_seeds: &[], + mock_dbt_refs: &[], + mock_table_functions: &[], + } +} + +/// `message @ cte` or `message @ cte: call`. +pub(crate) fn problem_summary(problem: &SqlTestHelperProblem) -> String { + problem.call.as_ref().map_or_else( + || format!("{} @ {}", problem.message, problem.cte_name), + |call| format!("{} @ {}: {call}", problem.message, problem.cte_name), + ) +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_sql_scenario_extraction.rs similarity index 95% rename from crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs rename to crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_sql_scenario_extraction.rs index 363581a468..63891cc0bf 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_sql_scenario_extraction.rs @@ -1,7 +1,7 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::compiler::main::sql_scenario_extraction::extract_scenario_json; -use crate::compiler::tests::test_types::ScenarioExtractionTestCase; +use crate::sql_test_attachment::tests::helpers::scenario_json; +use crate::sql_test_attachment::tests::test_types::ScenarioExtractionTestCase; #[test] fn given_scenarios_when_extracting_natively_then_outcomes_are_returned() { @@ -85,8 +85,8 @@ fn given_scenarios_when_extracting_natively_then_outcomes_are_returned() { for test_case in test_cases { assert_eq!( - extract_scenario_json(test_case.sql, "tests/scenarios/orders.sql", &syntax), - Ok(test_case.expected_json.to_owned()), + scenario_json(test_case.sql, "tests/scenarios/orders.sql", &syntax), + test_case.expected_json, "{}", test_case.description ); diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_sql_test_attachment_checks.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_sql_test_attachment_checks.rs new file mode 100644 index 0000000000..a62e7a921c --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_sql_test_attachment_checks.rs @@ -0,0 +1,349 @@ +//! The native SQL-test attachment checks: helper reads, authored offsets and stored scans. + +use std::collections::HashSet; + +use crate::assembly::analysis_session::main::sql_test_helper_reads::sql_test_helper_reads; +use crate::sql_test_attachment::main::authored_test_offset::authored_test_offset; +use crate::sql_test_attachment::main::sql_test_reads::sql_test_reads; +use crate::sql_test_attachment::main::stored_sql_tests::stored_sql_tests; +use crate::sql_test_attachment::main::test_without_target_model::test_without_target_model; +use crate::sql_test_attachment::main::unresolvable_test_references::unresolvable_test_references; +use crate::sql_test_attachment::models::{SqlTestHelperProblem, SqlTestReadFacts}; +use crate::sql_test_attachment::tests::helpers::{ + generic_dash_syntax, model_test_ctes, problem_summary, +}; +use crate::sql_test_attachment::tests::test_types::{ + AsciiMatcher, AttachmentCheckTestCase, AuthoredOffsetTestCase, HelperReadsTestCase, + StoredScanTestCase, +}; + +#[test] +fn given_cte_bodies_when_reading_helper_keys_then_only_unqualified_unshadowed_reads_count() { + let keys: Vec = ["base", "helper", "mock_orders", "étapé"] + .iter() + .map(|key| (*key).to_owned()) + .collect(); + let test_cases = [ + HelperReadsTestCase { + description: "a plain read", + body: "SELECT * FROM base", + expected_reads: Some(&["base"]), + }, + HelperReadsTestCase { + description: "case-insensitive reads in first-read order", + body: "SELECT * FROM BASE JOIN helper h ON 1=1 JOIN base b2 ON 1=1", + expected_reads: Some(&["base", "helper"]), + }, + HelperReadsTestCase { + description: "qualified names are not helper reads", + body: "SELECT * FROM s.base JOIN db.s.helper ON 1=1", + expected_reads: Some(&[]), + }, + HelperReadsTestCase { + description: "a nested CTE shadows its name", + body: "WITH base AS (SELECT 1) SELECT * FROM base, helper", + expected_reads: Some(&["helper"]), + }, + HelperReadsTestCase { + description: "a subquery CTE shadows only inside it", + body: "SELECT * FROM (WITH Helper AS (SELECT 1) SELECT * FROM helper) x JOIN base ON 1=1", + expected_reads: Some(&["base"]), + }, + HelperReadsTestCase { + description: "no table", + body: "SELECT 1", + expected_reads: Some(&[]), + }, + HelperReadsTestCase { + description: "an unrelated table", + body: "SELECT * FROM unrelated", + expected_reads: Some(&[]), + }, + HelperReadsTestCase { + description: "unparseable SQL", + body: "SELECT * FROM base WHERE (", + expected_reads: None, + }, + HelperReadsTestCase { + description: "a Unicode name folds", + body: "SELECT * FROM ÉTAPÉ", + expected_reads: Some(&["étapé"]), + }, + HelperReadsTestCase { + description: "a quoted name folds", + body: "SELECT * FROM \"Base\"", + expected_reads: Some(&["base"]), + }, + HelperReadsTestCase { + description: "set operations and subqueries", + body: "SELECT * FROM mock_orders UNION ALL SELECT * FROM helper WHERE id IN (SELECT id FROM base)", + expected_reads: Some(&["mock_orders", "helper", "base"]), + }, + HelperReadsTestCase { + description: "a column named like a helper", + body: "SELECT base FROM other", + expected_reads: Some(&[]), + }, + HelperReadsTestCase { + description: "several statements do not parse as one", + body: "SELECT * FROM helper; SELECT * FROM base", + expected_reads: None, + }, + HelperReadsTestCase { + description: "an EXISTS subquery", + body: "SELECT * FROM t WHERE EXISTS (SELECT 1 FROM helper)", + expected_reads: Some(&["helper"]), + }, + ]; + for test_case in test_cases { + let reads = sql_test_helper_reads(test_case.body, &keys).expect("the reads are computed"); + + assert_eq!( + reads, + test_case.expected_reads.map(|reads| reads + .iter() + .map(|read| (*read).to_owned()) + .collect::>()), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_discovered_block_when_mapping_body_offset_then_points_at_authored_text() { + let test_cases = [ + AuthoredOffsetTestCase { + description: "flush-left body after a comment that repeats the marker", + contents: "TEST ();\n-- zz_mark is named here\nWITH\nzz_mark AS (SELECT 1 AS a)\n", + authored: "-- zz_mark is named here\nWITH\nzz_mark AS (SELECT 1 AS a)", + span: Some((9, 65)), + body_offset: 30, + marker: "zz_mark AS", + expected_line_column: (4, 1), + }, + AuthoredOffsetTestCase { + description: "indented body with a tab before the marker", + contents: "TEST ();\n WITH\n h AS (SELECT 1 AS a),\n \tzz_mark AS (SELECT 1)\n", + authored: "WITH\nh AS (SELECT 1 AS a),\n zz_mark AS (SELECT 1)", + span: Some((13, 70)), + body_offset: 31, + marker: "zz_mark", + expected_line_column: (4, 6), + }, + AuthoredOffsetTestCase { + description: "tab inside a line before the marker", + contents: "TEST ();\nWITH\nh AS (SELECT 1\t AS a, 'é' AS zz_mark)\n", + authored: "WITH\nh AS (SELECT 1 AS a, 'é' AS zz_mark)", + span: Some((9, 51)), + body_offset: 35, + marker: "zz_mark", + expected_line_column: (3, 30), + }, + AuthoredOffsetTestCase { + description: "second block with blank lines after its header", + contents: "TEST (name \"one\");\nWITH a AS (SELECT 1)\n;\nTEST (name \"two\");\n\n\n WITH\n b AS (SELECT zz_mark)\n", + authored: "WITH\nb AS (SELECT zz_mark)", + span: Some((65, 93)), + body_offset: 18, + marker: "zz_mark", + expected_line_column: (8, 16), + }, + AuthoredOffsetTestCase { + description: "first body line indented on the header line", + contents: "TEST (); WITH zz_mark AS (SELECT 1)\n , b AS (SELECT 2)\n", + authored: "WITH zz_mark AS (SELECT 1)\n, b AS (SELECT 2)", + span: Some((11, 57)), + body_offset: 5, + marker: "zz_mark", + expected_line_column: (1, 17), + }, + ]; + for test_case in test_cases { + let offset: usize = authored_test_offset( + test_case.contents, + test_case.authored, + test_case.span, + test_case.body_offset, + ) + .expect("the offset is exact"); + let before: Vec = test_case.contents.chars().take(offset).collect(); + let marker: String = test_case + .contents + .chars() + .skip(offset) + .take(test_case.marker.chars().count()) + .collect(); + let line: usize = before + .iter() + .filter(|character| **character == '\n') + .count() + + 1; + let column: usize = offset + - before + .iter() + .rposition(|character| *character == '\n') + .map_or(0, |index| index + 1) + + 1; + + assert_eq!( + (marker.as_str(), (line, column)), + (test_case.marker, test_case.expected_line_column), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_scan_bytes_when_decoding_then_only_exact_shapes_are_reused() { + let model: &str = r#"{"kind":"model","mode":"model","authored":[["__ref__a","SELECT 1"]],"macroMocks":[],"mockModels":["a"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__b","SELECT 1"]],"expectedModels":["b"],"assertions":[],"assertionNames":[]}"#; + let test_cases = [ + StoredScanTestCase { + description: "an exact model result is reused", + value: format!("[{model}]"), + modes: &["model"], + expected_reused: true, + }, + StoredScanTestCase { + description: "undecodable bytes are rescanned", + value: "{]".to_owned(), + modes: &["model"], + expected_reused: false, + }, + StoredScanTestCase { + description: "a misshapen entry is rescanned", + value: r#"[{"mode": "model"}]"#.to_owned(), + modes: &["model"], + expected_reused: false, + }, + StoredScanTestCase { + description: "a different test count is rescanned", + value: format!("[{model}]"), + modes: &["model", "model"], + expected_reused: false, + }, + StoredScanTestCase { + description: "a changed mode is rescanned", + value: format!("[{model}]"), + modes: &["macro"], + expected_reused: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + stored_sql_tests(test_case.value.as_bytes(), test_case.modes).is_some(), + test_case.expected_reused, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_model_tests_when_checking_helpers_then_reads_and_problems_are_reported() { + let test_cases = [ + AttachmentCheckTestCase { + description: "an expected CTE reading a helper that reads an unmocked model", + authored: &[ + ("__source__raw", "SELECT 1 AS id"), + ("helper", "SELECT * FROM __ref(\"orders\")"), + ], + expected: &[("__expected__orders", "SELECT * FROM helper")], + assertions: &[], + mocks: (&[], &["raw"]), + expected_reads: (&["helper"], &["orders"]), + expected_problems: &[], + expected_without_target: "SQL test 'orders' reads no model, so the test has no model to run against @ __expected__orders", + }, + AttachmentCheckTestCase { + description: "an assertion calling an unmocked source", + authored: &[("__ref__stg", "SELECT 1 AS id")], + expected: &[], + assertions: &[("__assert__ok", "SELECT * FROM __source(\"raw\")")], + mocks: (&["stg"], &[]), + expected_reads: (&[], &[]), + expected_problems: &[ + "SQL test assertion CTE '__assert__ok' calls __source(\"raw\"), which the test does not mock, so the test query cannot resolve it @ __assert__ok: __source(\"raw\")", + ], + expected_without_target: "SQL test 'orders' reads no model, so the test has no model to run against @ __assert__ok", + }, + AttachmentCheckTestCase { + description: "an expected CTE reading an unknown model", + authored: &[("__source__raw", "SELECT 1 AS id")], + expected: &[("__expected__orders", "SELECT * FROM __ref(\"missing\")")], + assertions: &[], + mocks: (&[], &["raw"]), + expected_reads: (&[], &["missing"]), + expected_problems: &[ + "SQL test expected CTE '__expected__orders' references unknown model 'missing' @ __expected__orders: __ref(\"missing\")", + ], + expected_without_target: "SQL test 'orders' reads no model, so the test has no model to run against @ __expected__orders", + }, + AttachmentCheckTestCase { + description: "helpers reading each other", + authored: &[ + ("__source__raw", "SELECT 1 AS id"), + ("first", "SELECT * FROM second"), + ("second", "SELECT * FROM first"), + ], + expected: &[("__expected__orders", "SELECT * FROM first")], + assertions: &[], + mocks: (&[], &["raw"]), + expected_reads: (&["first", "second"], &[]), + expected_problems: &[ + "SQL test CTE 'first' reads itself through 'first' -> 'second' -> 'first', so the test query cannot define its CTEs in dependency order @ first", + ], + expected_without_target: "SQL test 'orders' reads no model, so the test has no model to run against @ __expected__orders", + }, + AttachmentCheckTestCase { + description: "an assertion reading only the mock of the model it tests", + authored: &[("__ref__orders", "SELECT 1 AS id")], + expected: &[], + assertions: &[("__assert__ok", "SELECT * FROM __ref__orders")], + mocks: (&["orders"], &[]), + expected_reads: (&[], &[]), + expected_problems: &[], + expected_without_target: "SQL test 'orders' mocks the model it tests (__ref__orders), so the test has no model to run against @ __assert__ok: __ref__orders", + }, + ]; + let syntax = generic_dash_syntax(); + let known: HashSet = ["orders".to_owned()].into_iter().collect(); + for test_case in test_cases { + let (authored, expected, assertions, mock_models, mock_sources) = ( + crate::sql_test_attachment::tests::helpers::cte_texts(test_case.authored), + crate::sql_test_attachment::tests::helpers::cte_texts(test_case.expected), + crate::sql_test_attachment::tests::helpers::cte_texts(test_case.assertions), + crate::sql_test_attachment::tests::helpers::owned(test_case.mocks.0), + crate::sql_test_attachment::tests::helpers::owned(test_case.mocks.1), + ); + let ctes = model_test_ctes( + (&authored, &expected, &assertions), + (&mock_models, &mock_sources), + ); + + let reads: SqlTestReadFacts = sql_test_reads(&ctes, &syntax, &AsciiMatcher).expect("reads"); + let problems: Vec = + unresolvable_test_references(&ctes, &known, &syntax, &AsciiMatcher).expect("problems"); + let without: SqlTestHelperProblem = + test_without_target_model(&ctes, "orders", &syntax, &AsciiMatcher).expect("problem"); + + assert_eq!( + ( + (reads.read_helper_names, reads.reference_target_model_names), + problems.iter().map(problem_summary).collect::>(), + problem_summary(&without), + ), + ( + ( + crate::sql_test_attachment::tests::helpers::owned(test_case.expected_reads.0), + crate::sql_test_attachment::tests::helpers::owned(test_case.expected_reads.1), + ), + crate::sql_test_attachment::tests::helpers::owned(test_case.expected_problems), + test_case.expected_without_target.to_owned(), + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_sql_test_classification_cases.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_sql_test_classification_cases.rs new file mode 100644 index 0000000000..4efffd85da --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_sql_test_classification_cases.rs @@ -0,0 +1,1071 @@ +//! Classification cases moved from the deleted Python wrapper tests, with exact outcomes. + +use crate::sql_test_attachment::tests::helpers::{ + assertion_targets, batch_outcome, parity_payload, scenario_outcome, +}; +use crate::sql_test_attachment::tests::test_types::{ + AssertionTargetsTestCase, BatchClassificationTestCase, ExtractionParityTestCase, + ScenarioClassificationTestCase, +}; + +#[test] +fn given_sql_test_blocks_when_classifying_then_outcomes_are_exact() { + let test_cases = [ + BatchClassificationTestCase { + description: "model roles: extracts dbt ref mocks from dbt ref prefixed ctes", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __dbt_ref__orders AS (SELECT 1 AS order_id),\n __dbt_ref__stripe__payments AS (SELECT 1 AS payment_id),\n __expected__fact_orders AS (SELECT 1 AS order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__dbt_ref__orders","SELECT 1 AS order_id"],["__dbt_ref__stripe__payments","SELECT 1 AS payment_id"]],"macroMocks":[],"mockModels":[],"mockSources":[],"mockSeeds":[],"mockDbtRefs":["orders","stripe__payments"],"mockTableFunctions":[],"expected":[["__expected__fact_orders","SELECT 1 AS order_id"]],"expectedModels":["fact_orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts model ctes when the trailing select is omitted", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts model ctes ending with a semicolon and comment", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id);\n -- end of test", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts table function fixtures from model tests", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __table_fn__customer_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__table_fn__customer_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":["customer_orders"],"expected":[["__expected__orders","SELECT 1 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: allows empty fixture marker for model expected output", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT * FROM __empty_fixture())\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT * FROM __empty_fixture()"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts zero-row assertion ctes", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id, 10 AS amount),\n __assert__no_negative_orders AS (\n SELECT * FROM __ref(\"orders\") WHERE amount < 0\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id, 10 AS amount"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[],"expectedModels":[],"assertions":[["__assert__no_negative_orders","SELECT * FROM __ref(\"orders\") WHERE amount < 0"]],"assertionNames":["no_negative_orders"]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts mixed expected and assertion ctes", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id, 10 AS amount),\n __expected__orders AS (SELECT 1 AS order_id, 10 AS amount),\n __assert__no_negative_orders AS (\n SELECT * FROM __ref(\"orders\") WHERE amount < 0\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id, 10 AS amount"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id, 10 AS amount"]],"expectedModels":["orders"],"assertions":[["__assert__no_negative_orders","SELECT * FROM __ref(\"orders\") WHERE amount < 0"]],"assertionNames":["no_negative_orders"]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: allows expected result and assertion to share an independent helper", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n helper AS (SELECT 1 AS order_id),\n __source__raw_orders AS (SELECT order_id FROM helper),\n __expected__orders AS (SELECT order_id FROM helper),\n __assert__positive_order_id AS (SELECT order_id FROM helper WHERE order_id < 1)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["helper","SELECT 1 AS order_id"],["__source__raw_orders","SELECT order_id FROM helper"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT order_id FROM helper"]],"expectedModels":["orders"],"assertions":[["__assert__positive_order_id","SELECT order_id FROM helper WHERE order_id < 1"]],"assertionNames":["positive_order_id"]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts seed mocks from seed-prefixed ctes", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __seed__country_codes AS (SELECT 'US' AS country_code),\n __expected__orders AS (SELECT 'US' AS country_code)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__seed__country_codes","SELECT 'US' AS country_code"]],"macroMocks":[],"mockModels":[],"mockSources":[],"mockSeeds":["country_codes"],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 'US' AS country_code"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts macro mocks from single string literal ctes", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __macro__country AS (SELECT '''US'''),\n __macro__empty AS (SELECT ''),\n __macro__literal_text AS (SELECT ''' + x + '''),\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[["country","'US'"],["empty",""],["literal_text","' + x + '"]],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts ctes from compact single line test sql", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n helper AS (SELECT 1 AS order_id),\n __ref__orders AS (SELECT * FROM helper),\n __expected__order_items AS (SELECT order_id FROM __ref__orders)\n SELECT 1;", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["helper","SELECT 1 AS order_id"],["__ref__orders","SELECT * FROM helper"]],"macroMocks":[],"mockModels":["orders"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__order_items","SELECT order_id FROM __ref__orders"]],"expectedModels":["order_items"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts ctes with comments strings and nested parentheses", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH /* leading */ helper_orders(order_id, note) AS (\n SELECT\n CAST('value ) inside string' AS VARCHAR) AS note,\n COALESCE(order_id, 0) AS order_id\n FROM raw_orders -- comment with ) and ,\n ) ,\n __source__raw_orders AS (SELECT * FROM helper_orders),\n __expected__orders AS (\n SELECT order_id, note FROM __source__raw_orders WHERE note != 'not ) done'\n )\n -- final comment before ceremonial select\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["helper_orders","SELECT\n CAST('value ) inside string' AS VARCHAR) AS note,\n COALESCE(order_id, 0) AS order_id\n FROM raw_orders -- comment with ) and ,"],["__source__raw_orders","SELECT * FROM helper_orders"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT order_id, note FROM __source__raw_orders WHERE note != 'not ) done'"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts multiple expected targets with mixed whitespace", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS(SELECT 1 AS order_id),__expected__orders AS(\n SELECT 1 AS order_id\n ),\n __expected__daily_revenue AS (SELECT SUM(1) AS revenue)\n SELECT 1;", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id"],["__expected__daily_revenue","SELECT SUM(1) AS revenue"]],"expectedModels":["orders","daily_revenue"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts ctes through nested helper chains and awkward comma placement", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH source_seed(seed_id, payload) AS (\n SELECT * FROM (\n SELECT 1 AS seed_id, 'a,b,c' AS payload\n UNION ALL\n SELECT 2 AS seed_id, 'literal with ) and , chars' AS payload\n ) AS nested_seed\n )/* comma after block comment */,\n normalized_orders AS(\n SELECT seed_id AS order_id, CONCAT(payload, ')') AS note FROM source_seed\n )\n ,__source__raw_orders AS(\n SELECT * FROM normalized_orders WHERE note NOT IN ('x,y', 'z)')\n ), expected_rows AS (\n SELECT order_id, note FROM __source__raw_orders\n WHERE order_id IN (SELECT seed_id FROM source_seed WHERE seed_id > 0)\n ),\n __expected__orders AS (SELECT order_id, note FROM expected_rows)\n SELECT 1;", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["source_seed","SELECT * FROM (\n SELECT 1 AS seed_id, 'a,b,c' AS payload\n UNION ALL\n SELECT 2 AS seed_id, 'literal with ) and , chars' AS payload\n ) AS nested_seed"],["normalized_orders","SELECT seed_id AS order_id, CONCAT(payload, ')') AS note FROM source_seed"],["__source__raw_orders","SELECT * FROM normalized_orders WHERE note NOT IN ('x,y', 'z)')"],["expected_rows","SELECT order_id, note FROM __source__raw_orders\n WHERE order_id IN (SELECT seed_id FROM source_seed WHERE seed_id > 0)"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT order_id, note FROM expected_rows"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: extracts ctes with inline comments around separators", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH helper_a AS (SELECT 1 AS order_id) -- comment before comma\n , helper_b AS (SELECT order_id FROM helper_a /* ) , */)\n , __ref__orders AS (SELECT order_id FROM helper_b), -- trailing comma comment\n __expected__orders AS (\n SELECT order_id FROM __ref__orders WHERE order_id = (SELECT MAX(order_id) FROM helper_b)\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["helper_a","SELECT 1 AS order_id"],["helper_b","SELECT order_id FROM helper_a /* ) , */"],["__ref__orders","SELECT order_id FROM helper_b"]],"macroMocks":[],"mockModels":["orders"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT order_id FROM __ref__orders WHERE order_id = (SELECT MAX(order_id) FROM helper_b)"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: allows explicit union branches in expected ctes", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (\n SELECT 1 AS order_id, 'new' AS status\n ),\n __expected__orders AS (\n SELECT 1 AS order_id, 'new' AS status\n UNION ALL\n SELECT 2 AS order_id, 'paid' AS status\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id, 'new' AS status"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id, 'new' AS status\n UNION ALL\n SELECT 2 AS order_id, 'paid' AS status"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: allows plain union with matching aliases in expected ctes", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (\n SELECT 1 AS order_id, 'created' AS status\n UNION\n SELECT 2 AS order_id, 'paid' AS status\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id, 'created' AS status\n UNION\n SELECT 2 AS order_id, 'paid' AS status"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: allows cast expressions with explicit aliases in expected ctes", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (\n SELECT\n CAST(1 AS INTEGER) AS order_id,\n CAST('paid' AS VARCHAR) AS status\n UNION ALL\n SELECT\n CAST(2 AS INTEGER) AS order_id,\n CAST('created' AS VARCHAR) AS status\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT\n CAST(1 AS INTEGER) AS order_id,\n CAST('paid' AS VARCHAR) AS status\n UNION ALL\n SELECT\n CAST(2 AS INTEGER) AS order_id,\n CAST('created' AS VARCHAR) AS status"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "model roles: allows bare identifiers in expected cte projections", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH expected_rows AS (SELECT 1 AS order_id, 'paid' AS status),\n __source__raw_orders AS (SELECT * FROM expected_rows),\n __expected__orders AS (\n SELECT order_id, status FROM expected_rows\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["expected_rows","SELECT 1 AS order_id, 'paid' AS status"],["__source__raw_orders","SELECT * FROM expected_rows"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT order_id, status FROM expected_rows"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "direct roles: extracts unsuffixed macro actual expected ctes in macro mode", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH input_values AS (SELECT ' PAID ' AS raw_status),\n __macro_actual__ AS (\n SELECT @normalize_status(\"raw_status\") AS status FROM input_values\n ),\n __macro_expected__ AS (SELECT 'paid' AS status)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"macro","helpers":[["input_values","SELECT ' PAID ' AS raw_status"]],"actual":["__macro_actual__","SELECT @normalize_status(\"raw_status\") AS status FROM input_values"],"expected":["__macro_expected__","SELECT 'paid' AS status"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "direct roles: extracts unsuffixed udf actual expected ctes in udf mode", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH input_values AS (SELECT 1250 AS amount_cents),\n __udf_actual__ AS (\n SELECT __udf(\"format_cents\")(amount_cents) AS formatted FROM input_values\n ),\n __udf_expected__ AS (SELECT '$12.50' AS formatted)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"udf","helpers":[["input_values","SELECT 1250 AS amount_cents"]],"actual":["__udf_actual__","SELECT __udf(\"format_cents\")(amount_cents) AS formatted FROM input_values"],"expected":["__udf_expected__","SELECT '$12.50' AS formatted"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "direct roles: extracts unsuffixed table function actual expected ctes in table_fn mode", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (\n SELECT customer_id, order_id FROM __table_fn(\"customer_orders\")(42)\n ),\n __table_fn_expected__ AS (SELECT 42 AS customer_id, 1 AS order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"table_fn","helpers":[],"actual":["__table_fn_actual__","SELECT customer_id, order_id FROM __table_fn(\"customer_orders\")(42)"],"expected":["__table_fn_expected__","SELECT 42 AS customer_id, 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "direct roles: extracts macro ctes when the trailing select is omitted", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT @double(2) AS value),\n __macro_expected__ AS (SELECT 4 AS value)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"macro","helpers":[],"actual":["__macro_actual__","SELECT @double(2) AS value"],"expected":["__macro_expected__","SELECT 4 AS value"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when ceremonial select is wrapped in a final result cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id),\n test_result AS (SELECT 1)\n SELECT * FROM test_result", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when assertion depends directly on expected result", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id),\n __assert__expected_is_nonempty AS (SELECT * FROM __expected__orders)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' check CTE '__assert__expected_is_nonempty' must not depend on '__expected__orders'; expected results and assertions must be independent","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when unparsed assertion uses quoted expected result", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id),\n __assert__expected_is_nonempty AS (\n SELECT @normalize_order(order_id) FROM \"__expected__orders\"\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' check CTE '__assert__expected_is_nonempty' must not depend on '__expected__orders'; expected results and assertions must be independent","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when assertion defines a shadowing expected result", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id),\n __assert__empty AS (\n WITH __expected__orders AS (SELECT 2 AS order_id)\n SELECT * FROM __expected__orders WHERE order_id < 0\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' check CTE '__assert__empty' must not define expected result CTE '__expected__orders'; expected results and assertions must be independent","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when lone assertion defines a nested expected result", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __assert__empty AS (\n WITH __expected__orders AS (SELECT 2 AS order_id)\n SELECT order_id FROM __expected__orders WHERE order_id < 0\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' check CTE '__assert__empty' must not define expected result CTE '__expected__orders'; expected results and assertions must be independent","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when expected result depends directly on assertion", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __assert__positive_order_id AS (SELECT 1 AS order_id WHERE 1 < 1),\n __expected__orders AS (SELECT order_id FROM __assert__positive_order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' check CTE '__expected__orders' must not depend on '__assert__positive_order_id'; expected results and assertions must be independent","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when assertion depends indirectly on expected result", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id),\n expected_helper AS (SELECT * FROM __expected__orders),\n __assert__expected_is_nonempty AS (SELECT * FROM expected_helper)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' check CTE '__assert__expected_is_nonempty' must not depend on '__expected__orders' through 'expected_helper'; expected results and assertions must be independent","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when macro mock returns multiple columns", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro__country AS (SELECT 'US', 'CA'),\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' macro mock '__macro__country' must be a single SELECT string literal with no FROM, UNION, or additional columns","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when macro mock returns non string literal", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro__country AS (SELECT 123),\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' macro mock '__macro__country' must be a single SELECT string literal, for example SELECT '''US'''","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when macro mock reads from a table", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro__country AS (SELECT 'US' FROM values_table),\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' macro mock '__macro__country' must be a single SELECT string literal with no FROM, UNION, or additional columns","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when macro mock uses union", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro__country AS (SELECT 'US' UNION ALL SELECT 'CA'),\n __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' macro mock '__macro__country' must be a single SELECT string literal with no FROM, UNION, or additional columns","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when test sql does not start with top level with", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must declare mock CTEs and one __expected__ CTE in a top-level WITH clause","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when assertion cte omits assertion name", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1), __assert__ AS (SELECT 1)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use __assert__ to identify a target","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when cte body is unclosed", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id,\n __expected__orders AS (SELECT 1 AS order_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test contains an unclosed parenthesis","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when final select is not ceremonial select one", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1), __expected__orders AS (SELECT 1 AS order_id)\n SELECT 1 FROM __expected__orders", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when final result cte is not ceremonial select one", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n __source__raw_orders AS (SELECT 1),\n __expected__orders AS (SELECT 1),\n test_result AS (SELECT 2)\n SELECT * FROM test_result", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when fallback terminal read follows a set operation", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n \"__source__raw_orders\" AS (SELECT 1),\n \"__expected__orders\" AS (SELECT 1),\n test_result AS (SELECT 1)\n SELECT 2 UNION ALL SELECT * FROM test_result", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' CTE name \"__source__raw_orders\" must be an unquoted identifier of ASCII letters, digits and underscores","help":"rename the CTE, for example __source__raw_orders; quoted CTE names and names with $ or non-ASCII characters are not supported","token":"\"__source__raw_orders\"","tokenOffset":13}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises for quoted CTE names instead of re-parsing the test", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n \"__source__raw_orders\" AS MATERIALIZED (SELECT 1 AS order_id),\n \"__expected__orders\" AS (SELECT order_id FROM \"__source__raw_orders\")\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' CTE name \"__source__raw_orders\" must be an unquoted identifier of ASCII letters, digits and underscores","help":"rename the CTE, for example __source__raw_orders; quoted CTE names and names with $ or non-ASCII characters are not supported","token":"\"__source__raw_orders\"","tokenOffset":13}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises for materialized CTEs", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS MATERIALIZED (SELECT 1 AS order_id),\n __expected__orders AS (SELECT order_id FROM __source__raw_orders)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' CTE '__source__raw_orders' must not use AS MATERIALIZED; materialization hints are not supported in SQL test CTEs","help":"remove MATERIALIZED and write __source__raw_orders AS (...)","token":"MATERIALIZED","tokenOffset":29}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when expected cte omits target name", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1), __expected__ AS (SELECT 1)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use __expected__ to identify a target","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when expected cte uses select star", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT * FROM __source__raw_orders)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must not use SELECT * in __expected__orders CTEs","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: rejects empty fixture marker for direct logic expected output", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 'paid' AS status),\n __macro_expected__ AS (SELECT * FROM __empty_fixture())\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must not use SELECT * in __macro_expected__ CTEs","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when union branch projection order differs", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (\n SELECT 1 AS order_id, 'paid' AS status\n UNION ALL\n SELECT 'created' AS status, 2 AS order_id\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __expected__orders projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when union branch projection alias is missing", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (\n SELECT CAST(1 AS INTEGER) AS order_id, CAST('paid' AS VARCHAR) AS status\n UNION ALL\n SELECT CAST(2 AS INTEGER), CAST('created' AS VARCHAR) AS status\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must alias every non-trivial __expected__orders projection","help":"name the column with an alias, for example CAST(2 AS INTEGER) AS ; only a bare or qualified column reference may go unaliased","token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when non union projection alias is missing", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (\n SELECT CAST(1 AS INTEGER), CAST('paid' AS VARCHAR) AS status\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must alias every non-trivial __expected__orders projection","help":"name the column with an alias, for example CAST(1 AS INTEGER) AS ; only a bare or qualified column reference may go unaliased","token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when expected branch is not a select query", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id),\n __expected__orders AS (\n SELECT 1 AS order_id\n UNION ALL\n VALUES (2)\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must define each __expected__orders set-operation branch as a SELECT query","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when model mode includes macro actual cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT @normalize_status('paid') AS status),\n __macro_expected__ AS (SELECT 'paid' AS status)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'model' but defines macro-test CTE '__macro_actual__'; use TEST (mode macro)","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when model mode includes udf actual cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT __udf(\"format_cents\")(1250) AS formatted),\n __udf_expected__ AS (SELECT '$12.50' AS formatted)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'model' but defines UDF-test CTE '__udf_actual__'; use TEST (mode udf)","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when model mode includes table function actual cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT * FROM __table_fn(\"customer_orders\")(42)),\n __table_fn_expected__ AS (SELECT 42 AS customer_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'model' but defines table_fn-test CTE '__table_fn_actual__'; use TEST (mode table_fn)","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when macro mode includes model test cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id),\n __macro_actual__ AS (SELECT @normalize_status('paid') AS status),\n __macro_expected__ AS (SELECT 'paid' AS status)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'macro' but defines model-test CTE '__source__raw_orders'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when macro expected calls macro", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT @normalize_status('paid') AS status),\n __macro_expected__ AS (SELECT @normalize_status('paid') AS status)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'macro' CTE __macro_expected__ must not call macros","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when macro helper calls macro", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH helper AS (SELECT @normalize_status('paid') AS status),\n __macro_actual__ AS (SELECT status FROM helper),\n __macro_expected__ AS (SELECT 'paid' AS status)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'macro' helper CTE 'helper' must not call macros; call macros only in __macro_actual__","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when udf mode includes model test cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id),\n __udf_actual__ AS (SELECT __udf(\"format_cents\")(1250) AS formatted),\n __udf_expected__ AS (SELECT '$12.50' AS formatted)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'udf' but defines model-test CTE '__source__raw_orders'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when udf expected calls udf", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT __udf(\"format_cents\")(1250) AS formatted),\n __udf_expected__ AS (SELECT __udf(\"format_cents\")(1250) AS formatted)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'udf' CTE __udf_expected__ must not call udf; call reusable logic only in __udf_actual__","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when udf helper calls udf", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH helper AS (SELECT __udf(\"format_cents\")(1250) AS formatted),\n __udf_actual__ AS (SELECT formatted FROM helper),\n __udf_expected__ AS (SELECT '$12.50' AS formatted)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'udf' helper CTE 'helper' must not call udf; call reusable logic only in __udf_actual__","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when table function expected calls table function", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT * FROM __table_fn(\"customer_orders\")(42)),\n __table_fn_expected__ AS (\n SELECT customer_id FROM __table_fn(\"customer_orders\")(42)\n )\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'table_fn' CTE __table_fn_expected__ must not call table_fn; call reusable logic only in __table_fn_actual__","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "invalid: raises when table function helper calls table function", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH helper AS (SELECT * FROM __table_fn(\"customer_orders\")(42)),\n __table_fn_actual__ AS (SELECT * FROM helper),\n __table_fn_expected__ AS (SELECT 42 AS customer_id)\n SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'table_fn' helper CTE 'helper' must not call table_fn; call reusable logic only in __table_fn_actual__","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "malformed: sql test without a with clause keeps sql test wording", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "SELECT * FROM orders", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must declare mock CTEs and one __expected__ CTE in a top-level WITH clause","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "malformed: sql test with an invalid cte name keeps sql test wording", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH 1orders AS (SELECT 1) SELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' CTE name '1orders' must be an unquoted identifier of ASCII letters, digits and underscores","help":"rename the CTE, for example cte_1orders; quoted CTE names and names with $ or non-ASCII characters are not supported","token":"1orders","tokenOffset":5}}"####, + }, + BatchClassificationTestCase { + description: "malformed: sql test without a ceremonial select keeps sql test wording", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id), result AS (SELECT 1) SELECT * FROM result", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "independence: a check reading an expected CTE through a helper", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id), helper AS (SELECT order_id FROM __expected__orders), __assert__same AS (SELECT order_id FROM helper) SELECT 1", "fileLabel": "tests/unit/example.sql", "mode": "model"}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/example.sql' check CTE '__assert__same' must not depend on '__expected__orders' through 'helper'; expected results and assertions must be independent","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "independence: a check joining an expected CTE in a comma-separated FROM", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__orders AS (SELECT 1 AS id), __expected__orders AS (SELECT 1 AS id), __assert__valid AS (SELECT expected.id FROM __source__orders source, __expected__orders expected) SELECT 1", "fileLabel": "tests/unit/example.sql", "mode": "model"}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/example.sql' check CTE '__assert__valid' must not depend on '__expected__orders'; expected results and assertions must be independent","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "dialect: snowflake backslash-escaped quote before a marker", + request: r####"{"syntax": {"backslashEscapeQuotes": ["'"], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--", "//"]}, "tests": [{"sql": "WITH\n__ref__stg_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n__assert__named AS (\n SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n)\nSELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model"}, {"sql": "WITH __ref__a AS (SELECT 1 AS id), __expected__b AS (SELECT 1 AS id) SELECT 1", "fileLabel": "tests/unit/plain.sql", "mode": "model"}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__ref__stg_orders","SELECT 'O\\'Brien), (' AS customer_name"]],"macroMocks":[],"mockModels":["stg_orders"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[],"expectedModels":[],"assertions":[["__assert__named","SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"]],"assertionNames":["named"]},{"kind":"model","mode":"model","authored":[["__ref__a","SELECT 1 AS id"]],"macroMocks":[],"mockModels":["a"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__b","SELECT 1 AS id"]],"expectedModels":["b"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "dialect: bigquery hash comment and triple quotes around markers", + request: r####"{"syntax": {"backslashEscapeQuotes": ["\"", "'", "`"], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": true, "nestedBlockComments": false, "lineCommentPrefixes": ["#", "--"]}, "tests": [{"sql": "WITH\n__ref__stg_orders AS (SELECT '''it's ), (''' AS note), # ), (\n__assert__named AS (\n SELECT * FROM __ref(\"order_totals\") # __ref(\"ignored\")\n)\nSELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model"}, {"sql": "WITH __ref__a AS (SELECT 1 AS id), __expected__b AS (SELECT 1 AS id) SELECT 1", "fileLabel": "tests/unit/plain.sql", "mode": "model"}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__ref__stg_orders","SELECT '''it's ), (''' AS note"]],"macroMocks":[],"mockModels":["stg_orders"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[],"expectedModels":[],"assertions":[["__assert__named","SELECT * FROM __ref(\"order_totals\") # __ref(\"ignored\")"]],"assertionNames":["named"]},{"kind":"model","mode":"model","authored":[["__ref__a","SELECT 1 AS id"]],"macroMocks":[],"mockModels":["a"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__b","SELECT 1 AS id"]],"expectedModels":["b"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "dialect: databricks backslash-escaped quote before a marker", + request: r####"{"syntax": {"backslashEscapeQuotes": ["\"", "'"], "escapeStringPrefix": false, "rawStringPrefix": true, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n__ref__stg_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n__assert__named AS (\n SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n)\nSELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model"}, {"sql": "WITH __ref__a AS (SELECT 1 AS id), __expected__b AS (SELECT 1 AS id) SELECT 1", "fileLabel": "tests/unit/plain.sql", "mode": "model"}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__ref__stg_orders","SELECT 'O\\'Brien), (' AS customer_name"]],"macroMocks":[],"mockModels":["stg_orders"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[],"expectedModels":[],"assertions":[["__assert__named","SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"]],"assertionNames":["named"]},{"kind":"model","mode":"model","authored":[["__ref__a","SELECT 1 AS id"]],"macroMocks":[],"mockModels":["a"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__b","SELECT 1 AS id"]],"expectedModels":["b"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "dialect: duckdb escape string and nested comment before a marker", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": true, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n__ref__stg_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n/* outer /* inner */ ), ( */\n__assert__named AS (\n SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n)\nSELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model"}, {"sql": "WITH __ref__a AS (SELECT 1 AS id), __expected__b AS (SELECT 1 AS id) SELECT 1", "fileLabel": "tests/unit/plain.sql", "mode": "model"}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__ref__stg_orders","SELECT E'O\\'Brien), (' AS customer_name"]],"macroMocks":[],"mockModels":["stg_orders"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[],"expectedModels":[],"assertions":[["__assert__named","SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"]],"assertionNames":["named"]},{"kind":"model","mode":"model","authored":[["__ref__a","SELECT 1 AS id"]],"macroMocks":[],"mockModels":["a"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__b","SELECT 1 AS id"]],"expectedModels":["b"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "dialect: postgres escape string before a marker", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": true, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH\n__ref__stg_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n__assert__named AS (\n SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n)\nSELECT 1", "fileLabel": "tests/unit/orders.sql", "mode": "model"}, {"sql": "WITH __ref__a AS (SELECT 1 AS id), __expected__b AS (SELECT 1 AS id) SELECT 1", "fileLabel": "tests/unit/plain.sql", "mode": "model"}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__ref__stg_orders","SELECT E'O\\'Brien), (' AS customer_name"]],"macroMocks":[],"mockModels":["stg_orders"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[],"expectedModels":[],"assertions":[["__assert__named","SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"]],"assertionNames":["named"]},{"kind":"model","mode":"model","authored":[["__ref__a","SELECT 1 AS id"]],"macroMocks":[],"mockModels":["a"],"mockSources":[],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__b","SELECT 1 AS id"]],"expectedModels":["b"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro duplicate actual", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1), __macro_actual__ AS (SELECT 2)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' defines duplicate CTE '__macro_actual__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro duplicate expected", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_expected__ AS (SELECT 1 AS value), __macro_expected__ AS (SELECT 2 AS value)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' defines duplicate CTE '__macro_expected__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro model-test cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __ref__orders AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'macro' but defines model-test CTE '__ref__orders'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro udf-test cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'macro' but defines UDF-test CTE '__udf_actual__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro table_fn-test cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_expected__ AS (SELECT 1 AS value)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'macro' but defines table_fn-test CTE '__table_fn_expected__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro reserved helper name", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __missing__ AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' uses reserved helper CTE name '__missing__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro missing expected", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'macro' must define exactly one __macro_actual__ CTE and exactly one __macro_expected__ CTE","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro helper calls macro", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH helper AS (SELECT @normalize_status('paid') AS status), __macro_actual__ AS (SELECT 1 AS status), __macro_expected__ AS (SELECT 1 AS status)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'macro' helper CTE 'helper' must not call macros; call macros only in __macro_actual__","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro expected calls macro", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS status), __macro_expected__ AS (SELECT @normalize_status('paid') AS status)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'macro' CTE __macro_expected__ must not call macros","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: macro expected unaliased projection names the expected cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS status), __macro_expected__ AS (SELECT 1 + 1)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must alias every non-trivial __macro_expected__ projection","help":"name the column with an alias, for example 1 + 1 AS ; only a bare or qualified column reference may go unaliased","token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: table_fn expected branch mismatch names the expected cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS value), __table_fn_expected__ AS (SELECT 1 AS value UNION ALL SELECT 2 AS other)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __table_fn_expected__ projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: udf duplicate actual", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1), __udf_actual__ AS (SELECT 2)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' defines duplicate CTE '__udf_actual__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: udf duplicate expected", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_expected__ AS (SELECT 1 AS value), __udf_expected__ AS (SELECT 2 AS value)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' defines duplicate CTE '__udf_expected__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: udf model-test cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __expected__orders AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'udf' but defines model-test CTE '__expected__orders'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: udf macro-test cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_expected__ AS (SELECT 1 AS value)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'udf' but defines macro-test CTE '__macro_expected__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: udf table_fn-test cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'udf' but defines table_fn-test CTE '__table_fn_actual__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: udf reserved helper name", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __actual AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' uses reserved helper CTE name '__actual'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: udf missing actual", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_expected__ AS (SELECT 1 AS value)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'udf' must define exactly one __udf_actual__ CTE and exactly one __udf_expected__ CTE","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: udf helper calls macro", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH helper AS (SELECT @normalize_status('paid') AS status), __udf_actual__ AS (SELECT 1 AS formatted), __udf_expected__ AS (SELECT 1 AS formatted)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'udf' helper CTE 'helper' must not call macros","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: udf expected calls udf", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS formatted), __udf_expected__ AS (SELECT __udf(\"format_cents\")(1250) AS formatted)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'udf' CTE __udf_expected__ must not call udf; call reusable logic only in __udf_actual__","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: table_fn duplicate actual", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1), __table_fn_actual__ AS (SELECT 2)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' defines duplicate CTE '__table_fn_actual__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: table_fn duplicate expected", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_expected__ AS (SELECT 1 AS value), __table_fn_expected__ AS (SELECT 2 AS value)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' defines duplicate CTE '__table_fn_expected__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: table_fn model-test cte", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'table_fn' but defines model-test CTE '__source__raw_orders'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: table_fn macro cte uses generic direct-logic wording", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'table_fn' but defines another direct-logic CTE '__macro_actual__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: table_fn udf cte uses generic direct-logic wording", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_expected__ AS (SELECT 1 AS value)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' is mode 'table_fn' but defines another direct-logic CTE '__udf_expected__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: table_fn reserved helper name", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __unexpected__ AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' uses reserved helper CTE name '__unexpected__'","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "direct logic: table_fn missing both", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH helper AS (SELECT 1)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' mode 'table_fn' must define exactly one __table_fn_actual__ CTE and exactly one __table_fn_expected__ CTE","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation: model: SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: model: SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: model: SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: model: SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: model: SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: model: SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: model: SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"model","mode":"model","authored":[["__source__raw_orders","SELECT 1 AS order_id"]],"macroMocks":[],"mockModels":[],"mockSources":["raw_orders"],"mockSeeds":[],"mockDbtRefs":[],"mockTableFunctions":[],"expected":[["__expected__orders","SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id"]],"expectedModels":["orders"],"assertions":[],"assertionNames":[]}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: macro: SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"macro","helpers":[],"actual":["__macro_actual__","SELECT 1 AS order_id"],"expected":["__macro_expected__","SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: macro: SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"macro","helpers":[],"actual":["__macro_actual__","SELECT 1 AS order_id"],"expected":["__macro_expected__","SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: macro: SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"macro","helpers":[],"actual":["__macro_actual__","SELECT 1 AS order_id"],"expected":["__macro_expected__","SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: macro: SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"macro","helpers":[],"actual":["__macro_actual__","SELECT 1 AS order_id"],"expected":["__macro_expected__","SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: macro: SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"macro","helpers":[],"actual":["__macro_actual__","SELECT 1 AS order_id"],"expected":["__macro_expected__","SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: macro: SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"macro","helpers":[],"actual":["__macro_actual__","SELECT 1 AS order_id"],"expected":["__macro_expected__","SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: macro: SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"macro","helpers":[],"actual":["__macro_actual__","SELECT 1 AS order_id"],"expected":["__macro_expected__","SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: udf: SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"udf","helpers":[],"actual":["__udf_actual__","SELECT 1 AS order_id"],"expected":["__udf_expected__","SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: udf: SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"udf","helpers":[],"actual":["__udf_actual__","SELECT 1 AS order_id"],"expected":["__udf_expected__","SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: udf: SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"udf","helpers":[],"actual":["__udf_actual__","SELECT 1 AS order_id"],"expected":["__udf_expected__","SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: udf: SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"udf","helpers":[],"actual":["__udf_actual__","SELECT 1 AS order_id"],"expected":["__udf_expected__","SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: udf: SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"udf","helpers":[],"actual":["__udf_actual__","SELECT 1 AS order_id"],"expected":["__udf_expected__","SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: udf: SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"udf","helpers":[],"actual":["__udf_actual__","SELECT 1 AS order_id"],"expected":["__udf_expected__","SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: udf: SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"udf","helpers":[],"actual":["__udf_actual__","SELECT 1 AS order_id"],"expected":["__udf_expected__","SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: table_fn: SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"table_fn","helpers":[],"actual":["__table_fn_actual__","SELECT 1 AS order_id"],"expected":["__table_fn_expected__","SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: table_fn: SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"table_fn","helpers":[],"actual":["__table_fn_actual__","SELECT 1 AS order_id"],"expected":["__table_fn_expected__","SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: table_fn: SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"table_fn","helpers":[],"actual":["__table_fn_actual__","SELECT 1 AS order_id"],"expected":["__table_fn_expected__","SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: table_fn: SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"table_fn","helpers":[],"actual":["__table_fn_actual__","SELECT 1 AS order_id"],"expected":["__table_fn_expected__","SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: table_fn: SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"table_fn","helpers":[],"actual":["__table_fn_actual__","SELECT 1 AS order_id"],"expected":["__table_fn_expected__","SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: table_fn: SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"table_fn","helpers":[],"actual":["__table_fn_actual__","SELECT 1 AS order_id"],"expected":["__table_fn_expected__","SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation: table_fn: SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"tests":[{"kind":"direct","mode":"table_fn","helpers":[],"actual":["__table_fn_actual__","SELECT 1 AS order_id"],"expected":["__table_fn_expected__","SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id"],"invalidCalls":false}]}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: model: SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __expected__orders projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: model: SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __expected__orders projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: model: SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id)", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __expected__orders projection names and order in every set-operation branch; branch 3 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: model: SELECT 1 AS order_id EXCEPT VALUES (1)", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id EXCEPT VALUES (1))", "fileLabel": "tests/unit/orders.sql", "mode": "model", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must define each __expected__orders set-operation branch as a SELECT query","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: macro: SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __macro_expected__ projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: macro: SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __macro_expected__ projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: macro: SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id)", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __macro_expected__ projection names and order in every set-operation branch; branch 3 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: macro: SELECT 1 AS order_id EXCEPT VALUES (1)", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __macro_actual__ AS (SELECT 1 AS order_id), __macro_expected__ AS (SELECT 1 AS order_id EXCEPT VALUES (1))", "fileLabel": "tests/unit/orders.sql", "mode": "macro", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must define each __macro_expected__ set-operation branch as a SELECT query","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: udf: SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __udf_expected__ projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: udf: SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __udf_expected__ projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: udf: SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id)", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __udf_expected__ projection names and order in every set-operation branch; branch 3 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: udf: SELECT 1 AS order_id EXCEPT VALUES (1)", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __udf_actual__ AS (SELECT 1 AS order_id), __udf_expected__ AS (SELECT 1 AS order_id EXCEPT VALUES (1))", "fileLabel": "tests/unit/orders.sql", "mode": "udf", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must define each __udf_expected__ set-operation branch as a SELECT query","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: table_fn: SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __table_fn_expected__ projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: table_fn: SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __table_fn_expected__ projection names and order in every set-operation branch; branch 2 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: table_fn: SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id)", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must use the same __table_fn_expected__ projection names and order in every set-operation branch; branch 3 does not match branch 1","help":null,"token":null,"tokenOffset":null}}"####, + }, + BatchClassificationTestCase { + description: "set operation rejected: table_fn: SELECT 1 AS order_id EXCEPT VALUES (1)", + request: r####"{"syntax": {"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}, "tests": [{"sql": "WITH __table_fn_actual__ AS (SELECT 1 AS order_id), __table_fn_expected__ AS (SELECT 1 AS order_id EXCEPT VALUES (1))", "fileLabel": "tests/unit/orders.sql", "mode": "table_fn", "raw": true}]}"####, + expected_response: r####"{"error":{"index":0,"message":"SQL test 'tests/unit/orders.sql' must define each __table_fn_expected__ set-operation branch as a SELECT query","help":null,"token":null,"tokenOffset":null}}"####, + }, + ]; + for test_case in test_cases { + assert_eq!( + batch_outcome(test_case.request), + test_case.expected_response, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_macro_free_sql_tests_when_classifying_before_and_after_expansion_then_payloads_match() { + let test_cases = [ + ExtractionParityTestCase { + description: "model roles retain authored ordering and independent checks", + sql: "WITH helper AS (SELECT 1 AS order_id), __source__raw_orders AS (SELECT order_id FROM helper), __macro__status AS (SELECT '''created'''), __expected__orders AS (SELECT order_id FROM helper), __assert__positive AS (SELECT order_id FROM helper WHERE order_id < 0) SELECT 1", + mode: "model", + expected_matches: true, + }, + ExtractionParityTestCase { + description: "macro direct logic retains helper actual and expected payloads", + sql: "WITH input AS (SELECT 'created' AS status), __macro_actual__ AS (SELECT @normalize_status(status) AS status FROM input), __macro_expected__ AS (SELECT 'created' AS status) SELECT 1", + mode: "macro", + expected_matches: true, + }, + ExtractionParityTestCase { + description: "udf direct logic retains helper actual and expected payloads", + sql: "WITH input AS (SELECT 1 AS value), __udf_actual__ AS (SELECT __udf(\"increment\")(value) AS value FROM input), __udf_expected__ AS (SELECT 2 AS value) SELECT 1", + mode: "udf", + expected_matches: true, + }, + ExtractionParityTestCase { + description: "table function direct logic retains actual and expected payloads", + sql: "WITH __table_fn_actual__ AS (SELECT order_id FROM __table_fn(\"customer_orders\")(1)), __table_fn_expected__ AS (SELECT 1 AS order_id) SELECT 1", + mode: "table_fn", + expected_matches: true, + }, + ExtractionParityTestCase { + description: "expected projection before where without from is accepted", + sql: "WITH __source__raw_orders AS (SELECT 1 AS id), __expected__orders AS (SELECT CAST(NULL AS INTEGER) AS id WHERE 1 = 0) SELECT 1", + mode: "model", + expected_matches: true, + }, + ExtractionParityTestCase { + description: "qualified expected projection uses the column name", + sql: "WITH __source__raw_orders AS (SELECT 1 AS id), sample AS (SELECT 1 AS id), __expected__orders AS (SELECT sample.id FROM sample) SELECT 1", + mode: "model", + expected_matches: true, + }, + ExtractionParityTestCase { + description: "clause-like qualified column uses its explicit alias", + sql: "WITH __source__raw_orders AS (SELECT 1 AS id), sample AS (SELECT 1 AS limit), __expected__orders AS (SELECT sample.limit AS id FROM sample) SELECT 1", + mode: "model", + expected_matches: true, + }, + ]; + for test_case in test_cases { + let raw = parity_payload(test_case.sql, test_case.mode, true); + let expanded = parity_payload(test_case.sql, test_case.mode, false); + + assert_eq!( + (raw.0 == expanded.0, raw.1), + (test_case.expected_matches, false), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_scenarios_when_classifying_then_outcomes_are_exact() { + let test_cases = [ + ScenarioClassificationTestCase { + description: "roles: extracts dbt ref fixtures", + sql: "WITH\n __dbt_ref__orders AS (SELECT 1 AS order_id),\n __dbt_ref__stripe__payments AS (SELECT 1 AS payment_id),\n __expected__fact_orders AS (SELECT 1 AS order_id)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__dbt_ref__orders","SELECT 1 AS order_id"],["__dbt_ref__stripe__payments","SELECT 1 AS payment_id"]],"expected":[["__expected__fact_orders","SELECT 1 AS order_id"]],"assertions":[],"sourceFixtures":[],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":["orders","stripe__payments"],"expectedModels":["fact_orders"],"assertionNames":[]}}"####, + }, + ScenarioClassificationTestCase { + description: "roles: extracts fixtures when the trailing select is omitted", + sql: "WITH\n __ref__stg_orders AS (SELECT 1 AS order_id),\n __expected__daily_revenue AS (SELECT 1 AS order_id)", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__ref__stg_orders","SELECT 1 AS order_id"]],"expected":[["__expected__daily_revenue","SELECT 1 AS order_id"]],"assertions":[],"sourceFixtures":[],"refFixtures":["stg_orders"],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["daily_revenue"],"assertionNames":[]}}"####, + }, + ScenarioClassificationTestCase { + description: "roles: extracts source ref seed fixtures expectations and assertions", + sql: "WITH\n helper_customers AS (SELECT 10 AS customer_id),\n __source__raw__orders AS (SELECT 1 AS order_id, 10 AS customer_id),\n __ref__stg_customers AS (SELECT customer_id FROM helper_customers),\n __seed__waffle_types AS (SELECT 1 AS waffle_type_id),\n __expected__daily_revenue AS (SELECT 1 AS order_id),\n __assert__no_negative_revenue AS (\n SELECT * FROM __ref(\"daily_revenue\") WHERE revenue < 0\n )\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["helper_customers","SELECT 10 AS customer_id"],["__source__raw__orders","SELECT 1 AS order_id, 10 AS customer_id"],["__ref__stg_customers","SELECT customer_id FROM helper_customers"],["__seed__waffle_types","SELECT 1 AS waffle_type_id"]],"expected":[["__expected__daily_revenue","SELECT 1 AS order_id"]],"assertions":[["__assert__no_negative_revenue","SELECT * FROM __ref(\"daily_revenue\") WHERE revenue < 0"]],"sourceFixtures":["raw__orders"],"refFixtures":["stg_customers"],"seedFixtures":["waffle_types"],"dbtRefFixtures":[],"expectedModels":["daily_revenue"],"assertionNames":["no_negative_revenue"]}}"####, + }, + ScenarioClassificationTestCase { + description: "roles: allows assertion only scenario target inference later", + sql: "WITH\n __source__raw__orders AS (SELECT 1 AS order_id),\n __assert__has_no_null_orders AS (\n SELECT * FROM __ref(\"fact_orders\") WHERE order_id IS NULL\n )\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__source__raw__orders","SELECT 1 AS order_id"]],"expected":[],"assertions":[["__assert__has_no_null_orders","SELECT * FROM __ref(\"fact_orders\") WHERE order_id IS NULL"]],"sourceFixtures":["raw__orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":[],"assertionNames":["has_no_null_orders"]}}"####, + }, + ScenarioClassificationTestCase { + description: "roles: allows expected only scenario with ref fixture", + sql: "WITH\n __ref__orders_base AS (SELECT 1 AS order_id),\n __expected__fact_orders AS (SELECT 1 AS order_id)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__ref__orders_base","SELECT 1 AS order_id"]],"expected":[["__expected__fact_orders","SELECT 1 AS order_id"]],"assertions":[],"sourceFixtures":[],"refFixtures":["orders_base"],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["fact_orders"],"assertionNames":[]}}"####, + }, + ScenarioClassificationTestCase { + description: "roles: allows expected only scenario with seed fixture", + sql: "WITH\n __seed__country_codes AS (SELECT 'US' AS country_code),\n __expected__dim_countries AS (SELECT 'US' AS country_code)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__seed__country_codes","SELECT 'US' AS country_code"]],"expected":[["__expected__dim_countries","SELECT 'US' AS country_code"]],"assertions":[],"sourceFixtures":[],"refFixtures":[],"seedFixtures":["country_codes"],"dbtRefFixtures":[],"expectedModels":["dim_countries"],"assertionNames":[]}}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when ceremonial select is wrapped in a final result cte", + sql: "WITH\n __ref__orders_base AS (SELECT 1 AS order_id),\n __expected__fact_orders AS (SELECT 1 AS order_id),\n scenario_result AS (SELECT 1)\n SELECT * FROM scenario_result", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them"}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when assertion depends directly on expected result", + sql: "WITH\n __source__raw__orders AS (SELECT 1 AS order_id),\n __expected__orders AS (SELECT 1 AS order_id),\n __assert__expected_is_nonempty AS (SELECT * FROM __expected__orders)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' check CTE '__assert__expected_is_nonempty' must not depend on '__expected__orders'; expected results and assertions must be independent"}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when expected result depends indirectly on assertion", + sql: "WITH\n __source__raw__orders AS (SELECT 1 AS order_id),\n __assert__positive_order_id AS (SELECT 1 AS order_id WHERE order_id < 1),\n assertion_helper AS (SELECT * FROM __assert__positive_order_id),\n __expected__orders AS (SELECT * FROM assertion_helper)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' check CTE '__expected__orders' must not depend on '__assert__positive_order_id' through 'assertion_helper'; expected results and assertions must be independent"}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when a scenario cte name is quoted", + sql: "WITH\n \"__source__raw__orders\" AS (SELECT 1 AS order_id),\n __expected__daily_revenue AS (SELECT 1 AS order_id)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' expected a CTE name"}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when a scenario cte uses a materialization hint", + sql: "WITH\n __source__raw__orders AS MATERIALIZED (SELECT 1 AS order_id),\n __expected__daily_revenue AS (SELECT 1 AS order_id)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' CTE '__source__raw__orders' must not use AS MATERIALIZED; materialization hints are not supported in SQL scenario CTEs"}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when scenario has no fixture ctes", + sql: "WITH __expected__fact_orders AS (SELECT 1 AS order_id)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' must define at least one __source__*, __ref__*, __seed__*, or __dbt_ref__* fixture CTE"}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when scenario has no expected or assertion ctes", + sql: "WITH __source__raw__orders AS (SELECT 1 AS order_id)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' must define at least one __expected__ or __assert__ CTE"}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when scenario uses macro mock cte", + sql: "WITH\n __source__raw__orders AS (SELECT 1 AS order_id),\n __macro__country AS (SELECT '''US'''),\n __expected__fact_orders AS (SELECT 1 AS order_id)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' does not support macro mock CTE '__macro__country'. Scenarios run real project macros; use SQL unit tests for macro mocks."}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when assertion cte suffix is missing", + sql: "WITH\n __source__raw__orders AS (SELECT 1 AS order_id),\n __assert__ AS (SELECT * FROM __ref(\"fact_orders\"))\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' must use __assert__ to identify a target"}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when duplicate cte names are present", + sql: "WITH\n __source__raw__orders AS (SELECT 1 AS order_id),\n __source__raw__orders AS (SELECT 2 AS order_id),\n __expected__fact_orders AS (SELECT 1 AS order_id)\n SELECT 1", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' defines duplicate CTE '__source__raw__orders'"}"####, + }, + ScenarioClassificationTestCase { + description: "invalid: raises when fallback terminal read follows a set operation", + sql: "WITH\n \"__ref__orders_base\" AS (SELECT 1 AS order_id),\n \"__expected__fact_orders\" AS (SELECT 1 AS order_id),\n scenario_result AS (SELECT 1)\n SELECT 2 UNION ALL SELECT * FROM scenario_result", + file_label: "tests/scenarios/revenue__customer_refund.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/revenue__customer_refund.sql' expected a CTE name"}"####, + }, + ScenarioClassificationTestCase { + description: "malformed: scenario without a with clause uses scenario wording", + sql: "SELECT * FROM orders", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/orders.sql' must declare fixture CTEs and at least one __expected__ or __assert__ CTE in a top-level WITH clause"}"####, + }, + ScenarioClassificationTestCase { + description: "malformed: unclosed comment before a fixture uses scenario wording", + sql: "WITH /* unfinished", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario contains an unclosed block comment"}"####, + }, + ScenarioClassificationTestCase { + description: "malformed: unclosed trailing comment uses scenario wording", + sql: "WITH __expected__orders AS (SELECT 1 AS order_id) SELECT 1; /* unfinished", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario contains an unclosed block comment"}"####, + }, + ScenarioClassificationTestCase { + description: "malformed: scenario with an invalid cte name uses scenario wording", + sql: "WITH 1orders AS (SELECT 1) SELECT 1", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/orders.sql' expected a CTE name"}"####, + }, + ScenarioClassificationTestCase { + description: "malformed: scenario without a ceremonial select uses scenario wording", + sql: "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id), result AS (SELECT 1) SELECT * FROM result", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/orders.sql' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them"}"####, + }, + ScenarioClassificationTestCase { + description: "malformed: scenario fixture without a target uses scenario wording", + sql: "WITH __source__ AS (SELECT 1 AS order_id), __expected__orders AS (SELECT 1 AS order_id) SELECT 1", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"error":"SQL scenario 'tests/scenarios/orders.sql' must use __source__ to identify a target"}"####, + }, + ScenarioClassificationTestCase { + description: "dialect: snowflake backslash-escaped quote before a marker", + sql: "WITH\n__source__raw_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n__assert__named AS (\n SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n)\nSELECT 1", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": ["'"], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--", "//"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__source__raw_orders","SELECT 'O\\'Brien), (' AS customer_name"]],"expected":[],"assertions":[["__assert__named","SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"]],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":[],"assertionNames":["named"]}}"####, + }, + ScenarioClassificationTestCase { + description: "dialect: bigquery hash comment and triple quotes around markers", + sql: "WITH\n__source__raw_orders AS (SELECT '''it's ), (''' AS note), # ), (\n__assert__named AS (\n SELECT * FROM __ref(\"order_totals\") # __ref(\"ignored\")\n)\nSELECT 1", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": ["\"", "'", "`"], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": true, "nestedBlockComments": false, "lineCommentPrefixes": ["#", "--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__source__raw_orders","SELECT '''it's ), (''' AS note"]],"expected":[],"assertions":[["__assert__named","SELECT * FROM __ref(\"order_totals\") # __ref(\"ignored\")"]],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":[],"assertionNames":["named"]}}"####, + }, + ScenarioClassificationTestCase { + description: "dialect: databricks backslash-escaped quote before a marker", + sql: "WITH\n__source__raw_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n__assert__named AS (\n SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n)\nSELECT 1", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": ["\"", "'"], "escapeStringPrefix": false, "rawStringPrefix": true, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__source__raw_orders","SELECT 'O\\'Brien), (' AS customer_name"]],"expected":[],"assertions":[["__assert__named","SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"]],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":[],"assertionNames":["named"]}}"####, + }, + ScenarioClassificationTestCase { + description: "dialect: duckdb escape string and nested comment before a marker", + sql: "WITH\n__source__raw_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n/* outer /* inner */ ), ( */\n__assert__named AS (\n SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n)\nSELECT 1", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": true, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__source__raw_orders","SELECT E'O\\'Brien), (' AS customer_name"]],"expected":[],"assertions":[["__assert__named","SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"]],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":[],"assertionNames":["named"]}}"####, + }, + ScenarioClassificationTestCase { + description: "dialect: postgres escape string before a marker", + sql: "WITH\n__source__raw_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n__assert__named AS (\n SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n)\nSELECT 1", + file_label: "tests/scenarios/orders.sql", + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": true, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}"####, + expected_outcome: r####"{"scenario":{"authored":[["__source__raw_orders","SELECT E'O\\'Brien), (' AS customer_name"]],"expected":[],"assertions":[["__assert__named","SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"]],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":[],"assertionNames":["named"]}}"####, + }, + ]; + for test_case in test_cases { + assert_eq!( + scenario_outcome(test_case.sql, test_case.file_label, test_case.syntax), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_dialect_assertions_when_reading_targets_then_only_active_refs_count() { + let test_cases = [ + AssertionTargetsTestCase { + description: "scenario snowflake backslash-escaped quote before a marker", + assertions: &["SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"], + syntax: r####"{"backslashEscapeQuotes": ["'"], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--", "//"]}"####, + expected_targets: &["order_totals"], + }, + AssertionTargetsTestCase { + description: "scenario bigquery hash comment and triple quotes around markers", + assertions: &["SELECT * FROM __ref(\"order_totals\") # __ref(\"ignored\")"], + syntax: r####"{"backslashEscapeQuotes": ["\"", "'", "`"], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": true, "nestedBlockComments": false, "lineCommentPrefixes": ["#", "--"]}"####, + expected_targets: &["order_totals"], + }, + AssertionTargetsTestCase { + description: "scenario databricks backslash-escaped quote before a marker", + assertions: &["SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"], + syntax: r####"{"backslashEscapeQuotes": ["\"", "'"], "escapeStringPrefix": false, "rawStringPrefix": true, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}"####, + expected_targets: &["order_totals"], + }, + AssertionTargetsTestCase { + description: "scenario duckdb escape string and nested comment before a marker", + assertions: &["SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"], + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": true, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}"####, + expected_targets: &["order_totals"], + }, + AssertionTargetsTestCase { + description: "scenario postgres escape string before a marker", + assertions: &["SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"], + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": true, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}"####, + expected_targets: &["order_totals"], + }, + AssertionTargetsTestCase { + description: "test snowflake backslash-escaped quote before a marker", + assertions: &["SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"], + syntax: r####"{"backslashEscapeQuotes": ["'"], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--", "//"]}"####, + expected_targets: &["order_totals"], + }, + AssertionTargetsTestCase { + description: "test bigquery hash comment and triple quotes around markers", + assertions: &["SELECT * FROM __ref(\"order_totals\") # __ref(\"ignored\")"], + syntax: r####"{"backslashEscapeQuotes": ["\"", "'", "`"], "escapeStringPrefix": false, "rawStringPrefix": false, "tripleQuotedStrings": true, "nestedBlockComments": false, "lineCommentPrefixes": ["#", "--"]}"####, + expected_targets: &["order_totals"], + }, + AssertionTargetsTestCase { + description: "test databricks backslash-escaped quote before a marker", + assertions: &["SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"], + syntax: r####"{"backslashEscapeQuotes": ["\"", "'"], "escapeStringPrefix": false, "rawStringPrefix": true, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}"####, + expected_targets: &["order_totals"], + }, + AssertionTargetsTestCase { + description: "test duckdb escape string and nested comment before a marker", + assertions: &["SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"], + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": true, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}"####, + expected_targets: &["order_totals"], + }, + AssertionTargetsTestCase { + description: "test postgres escape string before a marker", + assertions: &["SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")"], + syntax: r####"{"backslashEscapeQuotes": [], "escapeStringPrefix": true, "rawStringPrefix": false, "tripleQuotedStrings": false, "nestedBlockComments": true, "lineCommentPrefixes": ["--"]}"####, + expected_targets: &["order_totals"], + }, + ]; + for test_case in test_cases { + assert_eq!( + assertion_targets(test_case.assertions, test_case.syntax), + test_case.expected_targets, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_text_matching.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_text_matching.rs new file mode 100644 index 0000000000..60a3a8368b --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_text_matching.rs @@ -0,0 +1,149 @@ +use crate::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; +use crate::sql_test_attachment::models::{CteLocation, PythonTextMatcher}; +use crate::sql_test_attachment::tests::test_types::{ + CteLocationTestCase, ReadsWordTestCase, TokensTestCase, +}; +use crate::sql_test_attachment::types::SqlTestTextMatcher; + +#[test] +fn given_sql_when_listing_identifier_tokens_then_they_match_python_re_and_casefold() { + let test_cases = [ + TokensTestCase { + description: "tokens casefold and repeat once, dollar signs included", + sql: "select Orders, orders, ORDERS from x$y", + expected_tokens: &["select", "orders", "from", "x$y"], + }, + TokensTestCase { + description: "a token starts at an ASCII letter and continues through Unicode word text", + sql: "caf\u{e9} \u{e9}t\u{e9} _a1 1b", + expected_tokens: &["caf\u{e9}", "t\u{e9}", "_a1", "b"], + }, + TokensTestCase { + description: "casefolding expands sharp s and a long s cannot start a token", + sql: "Stra\u{df}e STRASSE \u{17f}pan", + expected_tokens: &["strasse", "pan"], + }, + ]; + for test_case in test_cases { + assert_eq!( + PythonTextMatcher.identifier_tokens(test_case.sql), + Ok(test_case + .expected_tokens + .iter() + .map(|token| (*token).to_owned()) + .collect::>()), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_when_searching_a_word_then_ignorecase_and_boundaries_match_python_re() { + let test_cases = [ + ReadsWordTestCase { + description: "case is ignored", + sql: "from orders o", + word: "ORDERS", + expected_reads: true, + }, + ReadsWordTestCase { + description: "a following word character is a longer word", + sql: "from orders_x", + word: "orders", + expected_reads: false, + }, + ReadsWordTestCase { + description: "a preceding dollar sign is part of the word", + sql: "$orders", + word: "orders", + expected_reads: false, + }, + ReadsWordTestCase { + description: "long s and the Kelvin sign match their ASCII letters", + sql: "\u{17f}pan \u{212a}ey", + word: "key", + expected_reads: true, + }, + ReadsWordTestCase { + description: "a preceding Unicode letter is a longer word", + sql: "caf\u{e9}orders", + word: "orders", + expected_reads: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + PythonTextMatcher.reads_word(test_case.sql, test_case.word), + Ok(test_case.expected_reads), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_helper_problems_when_locating_then_spans_match_the_python_locator() { + let test_cases = [ + CteLocationTestCase { + description: "the call inside the CTE is located", + contents: "WITH orders AS (SELECT * FROM __ref(\"orders\"))\nSELECT 1", + sql_body: "WITH orders", + cte_name: "orders", + call: Some("__ref(\"orders\")"), + expected_location: (1, 31, 1, 46), + }, + CteLocationTestCase { + description: "a quoted name, a line break and a non-breaking space locate the header", + contents: "WITH \"Orders\"\n as\u{a0}(select 1)\nSELECT 1", + sql_body: "WITH", + cte_name: "orders", + call: None, + expected_location: (1, 7, 1, 13), + }, + CteLocationTestCase { + description: "a dollar-prefixed name is skipped and a long s matches s", + contents: "TEST (name 't');\nWITH x$orders AS (1), \u{17f}ORDERS AS (\n __REF(\"orders\"))", + sql_body: "WITH", + cte_name: "sorders", + call: Some("__ref(\"orders\")"), + expected_location: (3, 3, 3, 18), + }, + CteLocationTestCase { + description: "a missing block and CTE fall back to the file start", + contents: "caf\u{e9}\nWITH b AS (1)", + sql_body: "missing body", + cte_name: "orders", + call: None, + expected_location: (1, 1, 1, 1), + }, + CteLocationTestCase { + description: "a call not found keeps the CTE name", + contents: "WITH orders AS(select 1)", + sql_body: "WITH", + cte_name: "orders", + call: Some("nothing"), + expected_location: (1, 6, 1, 12), + }, + ]; + for test_case in test_cases { + let location: CteLocation = sql_test_cte_location( + test_case.contents, + test_case.sql_body, + test_case.cte_name, + test_case.call, + ); + + assert_eq!( + ( + location.line, + location.column, + location.end_line, + location.end_column + ), + test_case.expected_location, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_types.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_types.rs new file mode 100644 index 0000000000..f3de69235e --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/tests/test_types.rs @@ -0,0 +1,98 @@ +pub(crate) struct ScenarioExtractionTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_json: &'static str, +} + +pub(crate) struct BatchClassificationTestCase { + pub(crate) description: &'static str, + pub(crate) request: &'static str, + pub(crate) expected_response: &'static str, +} + +pub(crate) struct ExtractionParityTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) mode: &'static str, + pub(crate) expected_matches: bool, +} + +pub(crate) struct ScenarioClassificationTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) file_label: &'static str, + pub(crate) syntax: &'static str, + pub(crate) expected_outcome: &'static str, +} + +pub(crate) struct AssertionTargetsTestCase { + pub(crate) description: &'static str, + pub(crate) assertions: &'static [&'static str], + pub(crate) syntax: &'static str, + pub(crate) expected_targets: &'static [&'static str], +} + +pub(crate) struct HelperReadsTestCase { + pub(crate) description: &'static str, + pub(crate) body: &'static str, + pub(crate) expected_reads: Option<&'static [&'static str]>, +} + +pub(crate) struct AuthoredOffsetTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) authored: &'static str, + pub(crate) span: Option<(usize, usize)>, + pub(crate) body_offset: usize, + pub(crate) marker: &'static str, + pub(crate) expected_line_column: (usize, usize), +} + +pub(crate) struct StoredScanTestCase { + pub(crate) description: &'static str, + pub(crate) value: String, + pub(crate) modes: &'static [&'static str], + pub(crate) expected_reused: bool, +} + +/// `(name, body)` CTEs of one model test. +pub(crate) type CteRows = &'static [(&'static str, &'static str)]; + +pub(crate) struct AttachmentCheckTestCase { + pub(crate) description: &'static str, + pub(crate) authored: CteRows, + pub(crate) expected: CteRows, + pub(crate) assertions: CteRows, + /// Mocked models and sources. + pub(crate) mocks: (&'static [&'static str], &'static [&'static str]), + /// Read helper names and reference target models. + pub(crate) expected_reads: (&'static [&'static str], &'static [&'static str]), + pub(crate) expected_problems: &'static [&'static str], + pub(crate) expected_without_target: &'static str, +} + +/// `[A-Za-z_][\\w$]*` matches for ASCII SQL, standing in for the host's Python `re`. +pub(crate) struct AsciiMatcher; + +pub(super) struct TokensTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_tokens: &'static [&'static str], +} + +pub(super) struct ReadsWordTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) word: &'static str, + pub(super) expected_reads: bool, +} + +pub(super) struct CteLocationTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) sql_body: &'static str, + pub(super) cte_name: &'static str, + pub(super) call: Option<&'static str>, + /// Line, column, end line and end column. + pub(super) expected_location: (usize, usize, usize, usize), +} diff --git a/crates/sqlbuild-analysis/src/sql_test_attachment/types.rs b/crates/sqlbuild-analysis/src/sql_test_attachment/types.rs new file mode 100644 index 0000000000..1ac092d753 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_test_attachment/types.rs @@ -0,0 +1,12 @@ +//! The text matching the SQL-test attachment checks are given by their host. + +/// The text pattern matches SQL-test attachment checks need, with the host's own semantics. +pub trait SqlTestTextMatcher { + type Error; + + /// Every `[A-Za-z_][\w$]*` identifier token of `sql`, casefolded, first occurrence first. + fn identifier_tokens(&self, sql: &str) -> Result, Self::Error>; + + /// Whether `sql` contains `word`, ignoring case, outside a longer `[\w$]` run. + fn reads_word(&self, sql: &str, word: &str) -> Result; +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs index 6036eb9bf5..df33fd5897 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/constants.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/constants.rs @@ -33,3 +33,13 @@ pub(crate) const PROJECTION_ENDS: [TokenType; 13] = [ TokenType::Except, TokenType::Intersect, ]; +/// The fingerprint dialect when a project names none. +pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; +/// Python's `NATIVE_DIALECT_ALIASES`. +pub(crate) const FINGERPRINT_DIALECT_ALIASES: [(&str, &str); 3] = [ + ("postgres", "postgresql"), + ("motherduck", "duckdb"), + ("sqlserver", "tsql"), +]; +/// The function language whose definitions are fingerprinted like queries. +pub(crate) const SQL_FUNCTION_LANGUAGE: &str = "sql"; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/compiler_query_fingerprint.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/compiler_query_fingerprint.rs new file mode 100644 index 0000000000..946c2aab59 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/compiler_query_fingerprint.rs @@ -0,0 +1,16 @@ +//! Python's `compute_query_hash`: a query fingerprint under the project's dialect alias. + +use crate::sql_tokens::constants::{FINGERPRINT_DIALECT_ALIASES, GENERIC_FINGERPRINT_DIALECT}; +use crate::sql_tokens::main::query_fingerprint::query_fingerprint; + +/// The fingerprint of `sql` under `dialect` (aliased, or generic when empty or absent). +pub fn compiler_query_fingerprint(sql: &str, dialect: Option<&str>) -> Result { + let dialect = match dialect.filter(|name| !name.is_empty()) { + None => GENERIC_FINGERPRINT_DIALECT, + Some(name) => FINGERPRINT_DIALECT_ALIASES + .iter() + .find(|(alias, _)| *alias == name) + .map_or(name, |(_, target)| target), + }; + query_fingerprint(sql, dialect) +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/definition_fingerprint.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/definition_fingerprint.rs new file mode 100644 index 0000000000..e121de21d9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/definition_fingerprint.rs @@ -0,0 +1,18 @@ +//! Python's `compute_function_definition_hash`. + +use sha2::{Digest, Sha256}; + +use crate::sql_tokens::constants::SQL_FUNCTION_LANGUAGE; +use crate::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint; + +/// SQL definitions fingerprinted like queries; other languages by the SHA-256 of their text. +pub fn definition_fingerprint( + sql: &str, + language: &str, + dialect: Option<&str>, +) -> Result { + if language == SQL_FUNCTION_LANGUAGE { + return compiler_query_fingerprint(sql, dialect); + } + Ok(format!("{:x}", Sha256::digest(sql.as_bytes()))) +} diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs index f073e8f1d7..1c05934a81 100644 --- a/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/mod.rs @@ -2,8 +2,11 @@ pub mod builtin_function_names; pub mod canonical_tokens; pub mod case_folding; +pub mod compiler_query_fingerprint; +pub mod definition_fingerprint; pub mod is_unquoted_word; pub mod name_positions; +pub mod normalized_query_text; pub mod projection_spans; pub mod query_fingerprint; pub mod token_texts; diff --git a/crates/sqlbuild-analysis/src/sql_tokens/main/normalized_query_text.rs b/crates/sqlbuild-analysis/src/sql_tokens/main/normalized_query_text.rs new file mode 100644 index 0000000000..5f36f98092 --- /dev/null +++ b/crates/sqlbuild-analysis/src/sql_tokens/main/normalized_query_text.rs @@ -0,0 +1,22 @@ +//! Python's `normalize_query_sql`: `re.sub(r"\s+", " ", sql.strip())`. + +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::python_strip::python_strip; + +/// `sql` stripped, with every run of Python whitespace collapsed to one space. +pub fn normalized_query_text(sql: &str) -> String { + let mut normalized = String::with_capacity(sql.len()); + let mut in_space = false; + for character in python_strip(sql).chars() { + if is_python_space(character) { + if !in_space { + normalized.push(' '); + } + in_space = true; + } else { + normalized.push(character); + in_space = false; + } + } + normalized +} diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/mod.rs b/crates/sqlbuild-attachments/src/audits/_helpers/mod.rs index 78767d0537..52f0b6ed7d 100644 --- a/crates/sqlbuild-attachments/src/audits/_helpers/mod.rs +++ b/crates/sqlbuild-attachments/src/audits/_helpers/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod parameters; pub(crate) mod policies; +pub(crate) mod references; pub(crate) mod rendering; +pub(crate) mod resources; diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/references.rs b/crates/sqlbuild-attachments/src/audits/_helpers/references.rs new file mode 100644 index 0000000000..9e1698ef85 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/_helpers/references.rs @@ -0,0 +1,38 @@ +//! The problem with one reference an audit body makes. + +use crate::audits::constants::{DBT_REF_KIND, REF_KIND, SEED_KIND, SOURCE_KIND}; +use crate::functions::models::KnownResources; + +/// What is wrong with the `kind` reference to `name`, or `None` when it resolves. +pub(crate) fn reference_problem( + kind: &str, + name: &str, + known: &KnownResources<'_>, +) -> Option { + match kind { + DBT_REF_KIND => Some(format!( + "may not use {}; audit dbt model checks belong in dbt", + example_call("__dbt_ref", name) + )), + REF_KIND if !known.models.contains(name) && known.seeds.contains(name) => Some(format!( + "references seed '{name}' with __ref(...). Use {} for seed references; __ref only \ + resolves models.", + example_call("__seed", name) + )), + REF_KIND if !known.models.contains(name) => { + Some(format!("references unknown model '{name}'")) + } + SEED_KIND if !known.seeds.contains(name) => { + Some(format!("references unknown seed '{name}'")) + } + SOURCE_KIND if !known.sources.contains(name) => { + Some(format!("references unknown source '{name}'")) + } + _ => None, + } +} + +/// `SqlReferenceKind.example_call(name, quote='"')`. +fn example_call(function: &str, name: &str) -> String { + format!("{function}(\"{}\")", name.replace('"', "\"\"")) +} diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/resources.rs b/crates/sqlbuild-attachments/src/audits/_helpers/resources.rs new file mode 100644 index 0000000000..61cfc18cc3 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/_helpers/resources.rs @@ -0,0 +1,48 @@ +//! The companion resources and single-resource problems of singular audits. + +use std::collections::BTreeSet; + +use crate::audits::constants::{ + FUNCTION_RESOURCE, SEED_KIND, SEED_RESOURCE, SOURCE_KIND, SOURCE_RESOURCE, TABLE_FUNCTION_KIND, +}; + +/// The resource kind a non-model reference kind names, if it counts as a companion. +pub(crate) fn companion_resource(kind: &str) -> Option<&'static str> { + match kind { + SOURCE_KIND => Some(SOURCE_RESOURCE), + SEED_KIND => Some(SEED_RESOURCE), + TABLE_FUNCTION_KIND => Some(FUNCTION_RESOURCE), + _ => None, + } +} + +/// `_single_resource_problem`: the message and help for an audit that is not cross-resource. +pub(crate) fn single_resource_problem( + models: &BTreeSet<&str>, + has_companions: bool, +) -> (String, String) { + if let (1, Some(model)) = (models.len(), models.iter().next()) { + return ( + format!("checks only model '{model}'; singular audits must be cross-resource"), + format!( + "attach a generic audit to '{model}' instead: write it in an audits/generic/ \ + role and select FROM @relation, joining @relation to itself for self-join checks" + ), + ); + } + if has_companions { + return ( + "checks only sources or seeds; singular audits must reference at least one model" + .to_owned(), + "attach YAML audits to the source or seed instead".to_owned(), + ); + } + ( + "references no SQLBuild resource; singular audits must reference two or more models, or \ + a model plus a source, seed, or table function via __ref(), __source(), or __seed()" + .to_owned(), + "reference resources through SQLBuild calls instead of hard-coded relation names, or \ + attach a generic audit to the resource being checked" + .to_owned(), + ) +} diff --git a/crates/sqlbuild-attachments/src/audits/constants.rs b/crates/sqlbuild-attachments/src/audits/constants.rs new file mode 100644 index 0000000000..b0bf059d59 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/constants.rs @@ -0,0 +1,15 @@ +//! Reference kinds audit validation reads and the cross-resource audit threshold. + +pub(crate) const REF_KIND: &str = "ref"; +pub(crate) const SEED_KIND: &str = "seed"; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const DBT_REF_KIND: &str = "dbt_ref"; +pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn"; +pub(crate) const MODEL_RESOURCE: &str = "model"; +pub(crate) const SOURCE_RESOURCE: &str = "source"; +pub(crate) const SEED_RESOURCE: &str = "seed"; +pub(crate) const FUNCTION_RESOURCE: &str = "function"; +/// Singular audits reading fewer models must also read a companion resource. +pub(crate) const CROSS_MODEL_MINIMUM: usize = 2; +/// The run scope Python falls back to without an instance or project value. +pub(crate) const FALLBACK_RUN_SCOPE: &str = "delta_and_final"; diff --git a/crates/sqlbuild-attachments/src/audits/main/attached_model_reference_error.rs b/crates/sqlbuild-attachments/src/audits/main/attached_model_reference_error.rs new file mode 100644 index 0000000000..979ed55e99 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/main/attached_model_reference_error.rs @@ -0,0 +1,24 @@ +//! `validate_model_attached_audit_references`: a model's audit must read that model. + +use crate::audits::constants::{MODEL_RESOURCE, REF_KIND}; + +/// The error when a model-attached audit's references (`(kind, name)`) skip the model. +#[must_use] +pub fn attached_model_reference_error( + audit_label: &str, + attached: (&str, &str), + references: &[(&str, &str)], +) -> Option { + let (kind, name) = attached; + if kind != MODEL_RESOURCE { + return None; + } + if references.iter().any(|&(reference_kind, reference_name)| { + reference_kind == REF_KIND && reference_name == name + }) { + return None; + } + Some(format!( + "{audit_label}: model-attached audit must reference the attached model '{name}' via __ref()" + )) +} diff --git a/crates/sqlbuild-attachments/src/audits/main/audit_reference_error.rs b/crates/sqlbuild-attachments/src/audits/main/audit_reference_error.rs new file mode 100644 index 0000000000..752b306112 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/main/audit_reference_error.rs @@ -0,0 +1,19 @@ +//! `validate_audit_references`: the error Python raises for an audit's bad reference. + +use crate::audits::_helpers::references::reference_problem; +use crate::functions::models::KnownResources; + +/// The error for the first of `references` (`(kind, name)`) the project cannot resolve. +#[must_use] +pub fn audit_reference_error( + relative_path: &str, + references: &[(&str, &str)], + known: &KnownResources<'_>, +) -> Option { + for &(kind, name) in references { + if let Some(problem) = reference_problem(kind, name, known) { + return Some(format!("Audit file {relative_path} {problem}")); + } + } + None +} diff --git a/crates/sqlbuild-attachments/src/audits/main/mod.rs b/crates/sqlbuild-attachments/src/audits/main/mod.rs index 24d931eb6d..1a8d416488 100644 --- a/crates/sqlbuild-attachments/src/audits/main/mod.rs +++ b/crates/sqlbuild-attachments/src/audits/main/mod.rs @@ -1 +1,6 @@ +pub mod attached_model_reference_error; +pub mod audit_reference_error; pub mod render_attached_audit; +pub mod render_parameterized_sql; +pub mod resolve_audit_policies; +pub mod singular_audit_resources; diff --git a/crates/sqlbuild-attachments/src/audits/main/render_parameterized_sql.rs b/crates/sqlbuild-attachments/src/audits/main/render_parameterized_sql.rs new file mode 100644 index 0000000000..f26cc18c19 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/main/render_parameterized_sql.rs @@ -0,0 +1,32 @@ +use crate::audits::_helpers::parameters::{RenderStop, parameter_matches}; +use crate::audits::models::{ArgumentValue, ParameterRenderStop}; + +/// `render_parameterized_sql`: raw and quoted arguments rendered into reusable SQL text. +pub fn render_parameterized_sql( + sql: &str, + arguments: &[(String, ArgumentValue)], + reject_unused: bool, +) -> Result { + if reject_unused { + let referenced: Vec<&str> = parameter_matches(sql) + .into_iter() + .map(|item| item.name) + .collect(); + let mut unused: Vec = arguments + .iter() + .map(|(name, _)| name.clone()) + .filter(|name| !referenced.contains(&name.as_str())) + .collect(); + unused.sort(); + unused.dedup(); + if !unused.is_empty() { + return Err(ParameterRenderStop::UnusedArguments(unused)); + } + } + crate::audits::_helpers::parameters::render_parameterized_sql(sql, arguments).map_err(|stop| { + match stop { + RenderStop::MissingArgument(name) => ParameterRenderStop::MissingArgument(name), + RenderStop::UnsupportedValue(name) => ParameterRenderStop::UnsupportedValue(name), + } + }) +} diff --git a/crates/sqlbuild-attachments/src/audits/main/resolve_audit_policies.rs b/crates/sqlbuild-attachments/src/audits/main/resolve_audit_policies.rs new file mode 100644 index 0000000000..6c7a042d90 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/main/resolve_audit_policies.rs @@ -0,0 +1,22 @@ +//! `resolve_audit_severity` then `resolve_audit_run_scope`, with Python's errors. + +use crate::audits::_helpers::policies::{resolve_run_scope, resolve_severity}; +use crate::audits::constants::FALLBACK_RUN_SCOPE; +use crate::audits::models::PolicySource; + +/// `(severity, run scope)`, or the first error resolving them. +pub fn resolve_audit_policies( + severity: (Option<&str>, Option<&str>, &str), + run_scope: (Option<&str>, Option<&str>), +) -> Result<(&'static str, String), String> { + let (instance_severity, default_severity, audit_label) = severity; + let severity: &'static str = + resolve_severity(instance_severity, default_severity, audit_label)?; + let (instance, default) = run_scope; + let run_scope: String = match resolve_run_scope(instance, default)? { + PolicySource::Instance => instance.unwrap_or_default().to_owned(), + PolicySource::Default => default.unwrap_or_default().to_owned(), + PolicySource::Fallback => FALLBACK_RUN_SCOPE.to_owned(), + }; + Ok((severity, run_scope)) +} diff --git a/crates/sqlbuild-attachments/src/audits/main/singular_audit_resources.rs b/crates/sqlbuild-attachments/src/audits/main/singular_audit_resources.rs new file mode 100644 index 0000000000..ac885ace53 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/main/singular_audit_resources.rs @@ -0,0 +1,37 @@ +//! `singular_audit_resources`: what a singular audit reads, and why it is not cross-resource. + +use std::collections::BTreeSet; + +use crate::audits::_helpers::resources::{companion_resource, single_resource_problem}; +use crate::audits::constants::{CROSS_MODEL_MINIMUM, MODEL_RESOURCE, REF_KIND}; +use crate::audits::models::SingularAuditResources; + +/// The sorted `(kind, name)` resources `references` read and the P004 problem, if any. +#[must_use] +pub fn singular_audit_resources(references: &[(&str, &str)]) -> SingularAuditResources { + let mut models: BTreeSet<&str> = BTreeSet::new(); + let mut companions: BTreeSet<(&'static str, &str)> = BTreeSet::new(); + for &(kind, name) in references { + if kind == REF_KIND { + models.insert(name); + } else if let Some(resource) = companion_resource(kind) { + companions.insert((resource, name)); + } + } + let mut resources: BTreeSet<(String, String)> = models + .iter() + .map(|name| (MODEL_RESOURCE.to_owned(), (*name).to_owned())) + .collect(); + resources.extend( + companions + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())), + ); + let problem = (models.len() < CROSS_MODEL_MINIMUM + && (models.is_empty() || companions.is_empty())) + .then(|| single_resource_problem(&models, !companions.is_empty())); + SingularAuditResources { + resources: resources.into_iter().collect(), + problem, + } +} diff --git a/crates/sqlbuild-attachments/src/audits/mod.rs b/crates/sqlbuild-attachments/src/audits/mod.rs index e6e5a52eb4..1e64f3d116 100644 --- a/crates/sqlbuild-attachments/src/audits/mod.rs +++ b/crates/sqlbuild-attachments/src/audits/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-attachments/src/audits/models.rs b/crates/sqlbuild-attachments/src/audits/models.rs index 9b5a709349..362a576a8a 100644 --- a/crates/sqlbuild-attachments/src/audits/models.rs +++ b/crates/sqlbuild-attachments/src/audits/models.rs @@ -56,3 +56,20 @@ pub enum AuditRendering { Rendered(RenderedAudit), Failed(String), } + +/// Why `render_parameterized_sql` stops. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ParameterRenderStop { + /// Passed arguments no parameter names, sorted. + UnusedArguments(Vec), + MissingArgument(String), + /// The argument whose value cannot be rendered. + UnsupportedValue(String), +} + +/// The resources a singular audit reads, sorted, and its P004 `(message, help)`, if any. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SingularAuditResources { + pub resources: Vec<(String, String)>, + pub problem: Option<(String, String)>, +} diff --git a/crates/sqlbuild-attachments/src/audits/tests.rs b/crates/sqlbuild-attachments/src/audits/tests.rs index 8c3f865211..f0cdefa7de 100644 --- a/crates/sqlbuild-attachments/src/audits/tests.rs +++ b/crates/sqlbuild-attachments/src/audits/tests.rs @@ -4,5 +4,9 @@ mod helpers; mod render_attached_audit; #[path = "tests/test_render_parameterized_sql.rs"] mod render_parameterized_sql; +#[path = "tests/test_resolve_audit_policies.rs"] +mod resolve_audit_policies; +#[path = "tests/test_singular_audit_resources.rs"] +mod singular_audit_resources; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-attachments/src/audits/tests/helpers.rs b/crates/sqlbuild-attachments/src/audits/tests/helpers.rs index c41c742aa8..392b4fd6d6 100644 --- a/crates/sqlbuild-attachments/src/audits/tests/helpers.rs +++ b/crates/sqlbuild-attachments/src/audits/tests/helpers.rs @@ -7,6 +7,10 @@ use crate::audits::models::{ /// Arguments shared by the parameter rendering cases. pub(super) fn order_arguments() -> Vec<(String, ArgumentValue)> { vec![ + ( + "label".to_owned(), + ArgumentValue::Text("O'Brien@orders".to_owned()), + ), ( "column".to_owned(), ArgumentValue::Text("status".to_owned()), diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs b/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs index b3e1bc6736..2528fc1520 100644 --- a/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs +++ b/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs @@ -1,10 +1,17 @@ use crate::audits::_helpers::parameters::RenderStop; +use crate::audits::main::render_parameterized_sql::render_parameterized_sql; +use crate::audits::models::{ArgumentValue, ParameterRenderStop}; use crate::audits::tests::helpers::rendered_sql; -use crate::audits::tests::test_types::ParameterizedSqlTestCase; +use crate::audits::tests::test_types::{ParameterizedSqlTestCase, UnusedArgumentsTestCase}; #[test] fn given_parameterized_sql_when_rendering_then_python_text_is_returned() { let test_cases = [ + ParameterizedSqlTestCase { + description: "quoted inserted argument markers are not rescanned", + sql: "SELECT @'label'", + expected_sql: Ok("SELECT 'O''Brien@orders'"), + }, ParameterizedSqlTestCase { description: "raw and quoted parameters: lists joined, only text quoted, quotes doubled", sql: "WHERE @column IN (@'values') AND x = @values", @@ -38,3 +45,43 @@ fn given_parameterized_sql_when_rendering_then_python_text_is_returned() { ); } } + +#[test] +fn given_unreferenced_arguments_when_rejecting_unused_then_their_sorted_names_stop_rendering() { + let test_cases = [ + UnusedArgumentsTestCase { + description: "unused names are reported sorted before a missing argument", + sql: "GRANT SELECT TO @'role' AND @missing", + arguments: &["scope", "role", "mode"], + expected_sql: Err(ParameterRenderStop::UnusedArguments(vec![ + "mode".to_owned(), + "scope".to_owned(), + ])), + }, + UnusedArgumentsTestCase { + description: "every argument referenced renders", + sql: "GRANT SELECT TO @'role'", + arguments: &["role"], + expected_sql: Ok("GRANT SELECT TO 'role-value'"), + }, + ]; + + for test_case in test_cases { + let arguments: Vec<(String, ArgumentValue)> = test_case + .arguments + .iter() + .map(|name| { + ( + (*name).to_owned(), + ArgumentValue::Text(format!("{name}-value")), + ) + }) + .collect(); + assert_eq!( + render_parameterized_sql(test_case.sql, &arguments, true), + test_case.expected_sql.map(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_resolve_audit_policies.rs b/crates/sqlbuild-attachments/src/audits/tests/test_resolve_audit_policies.rs new file mode 100644 index 0000000000..1b5508010e --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/tests/test_resolve_audit_policies.rs @@ -0,0 +1,47 @@ +use crate::audits::main::resolve_audit_policies::resolve_audit_policies; +use crate::audits::tests::test_types::AuditPoliciesTestCase; + +#[test] +fn given_audit_policies_when_resolving_then_instance_then_project_then_fallback_wins() { + let test_cases = [ + AuditPoliciesTestCase { + description: "nothing authored falls back", + severity: (None, None), + run_scope: (None, None), + expected_policies: Ok(("error", "delta_and_final")), + }, + AuditPoliciesTestCase { + description: "instance values override project defaults", + severity: (Some("warn"), Some("error")), + run_scope: (Some("final"), Some("delta_and_final")), + expected_policies: Ok(("warn", "final")), + }, + AuditPoliciesTestCase { + description: "project defaults apply without instance values", + severity: (None, Some("warn")), + run_scope: (None, Some("final")), + expected_policies: Ok(("warn", "final")), + }, + AuditPoliciesTestCase { + description: "an unknown instance run scope is rejected", + severity: (None, None), + run_scope: (Some("always"), None), + expected_policies: Err( + "unknown audit run_scope 'always'; valid values: delta_and_final, final", + ), + }, + ]; + + for test_case in test_cases { + let (instance, default) = test_case.severity; + assert_eq!( + resolve_audit_policies((instance, default, "audit 'floor'"), test_case.run_scope) + .as_ref() + .map(|(severity, run_scope)| (*severity, run_scope.as_str())) + .map_err(String::as_str), + test_case.expected_policies, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_singular_audit_resources.rs b/crates/sqlbuild-attachments/src/audits/tests/test_singular_audit_resources.rs new file mode 100644 index 0000000000..b054365c95 --- /dev/null +++ b/crates/sqlbuild-attachments/src/audits/tests/test_singular_audit_resources.rs @@ -0,0 +1,56 @@ +use crate::audits::main::singular_audit_resources::singular_audit_resources; +use crate::audits::tests::test_types::SingularResourcesTestCase; + +#[test] +fn given_singular_audit_references_when_collecting_resources_then_cross_resource_rule_applies() { + let test_cases = [ + SingularResourcesTestCase { + description: "two models are cross-resource", + references: vec![("ref", "orders"), ("ref", "customers"), ("ref", "orders")], + expected_resources: vec![("model", "customers"), ("model", "orders")], + expected_problem: None, + }, + SingularResourcesTestCase { + description: "a model and a source are cross-resource", + references: vec![("source", "raw_orders"), ("ref", "orders")], + expected_resources: vec![("model", "orders"), ("source", "raw_orders")], + expected_problem: None, + }, + SingularResourcesTestCase { + description: "one model alone names the model", + references: vec![("ref", "orders")], + expected_resources: vec![("model", "orders")], + expected_problem: Some( + "checks only model 'orders'; singular audits must be cross-resource", + ), + }, + SingularResourcesTestCase { + description: "sources and seeds alone need a model", + references: vec![("source", "raw_orders"), ("seed", "regions")], + expected_resources: vec![("seed", "regions"), ("source", "raw_orders")], + expected_problem: Some( + "checks only sources or seeds; singular audits must reference at least one model", + ), + }, + ]; + + for test_case in test_cases { + let resources = singular_audit_resources(&test_case.references); + assert_eq!( + resources + .resources + .iter() + .map(|(kind, name)| (kind.as_str(), name.as_str())) + .collect::>(), + test_case.expected_resources, + "{}", + test_case.description + ); + assert_eq!( + resources.problem.as_ref().map(|problem| problem.0.as_str()), + test_case.expected_problem, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_types.rs b/crates/sqlbuild-attachments/src/audits/tests/test_types.rs index f93c7734a4..5421d9df90 100644 --- a/crates/sqlbuild-attachments/src/audits/tests/test_types.rs +++ b/crates/sqlbuild-attachments/src/audits/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::audits::_helpers::parameters::RenderStop; -use crate::audits::models::{AuditAttachment, AuditRendering}; +use crate::audits::models::{AuditAttachment, AuditRendering, ParameterRenderStop}; pub(super) struct ParameterizedSqlTestCase { pub(super) description: &'static str, @@ -7,8 +7,30 @@ pub(super) struct ParameterizedSqlTestCase { pub(super) expected_sql: Result<&'static str, RenderStop>, } +pub(super) struct UnusedArgumentsTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// Argument names, each passed as the text `-value`. + pub(super) arguments: &'static [&'static str], + pub(super) expected_sql: Result<&'static str, ParameterRenderStop>, +} + pub(super) struct AttachedAuditTestCase { pub(super) description: &'static str, pub(super) attachment: AuditAttachment, pub(super) expected_rendering: AuditRendering, } + +pub(super) struct SingularResourcesTestCase { + pub(super) description: &'static str, + pub(super) references: Vec<(&'static str, &'static str)>, + pub(super) expected_resources: Vec<(&'static str, &'static str)>, + pub(super) expected_problem: Option<&'static str>, +} + +pub(super) struct AuditPoliciesTestCase { + pub(super) description: &'static str, + pub(super) severity: (Option<&'static str>, Option<&'static str>), + pub(super) run_scope: (Option<&'static str>, Option<&'static str>), + pub(super) expected_policies: Result<(&'static str, &'static str), &'static str>, +} diff --git a/crates/sqlbuild-attachments/src/functions/_helpers/mod.rs b/crates/sqlbuild-attachments/src/functions/_helpers/mod.rs index eff19e8c44..95f3176be1 100644 --- a/crates/sqlbuild-attachments/src/functions/_helpers/mod.rs +++ b/crates/sqlbuild-attachments/src/functions/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod header_parsing; pub(crate) mod namespace; +pub(crate) mod references; pub(crate) mod values; diff --git a/crates/sqlbuild-attachments/src/functions/_helpers/references.rs b/crates/sqlbuild-attachments/src/functions/_helpers/references.rs new file mode 100644 index 0000000000..4f254a80ef --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/_helpers/references.rs @@ -0,0 +1,51 @@ +//! The problem with one reference a SQL function body makes. + +use crate::functions::constants::{ + DBT_REF_KIND, REF_KIND, SEED_KIND, SOURCE_KIND, TABLE_FUNCTION_KIND, UDF_KIND, +}; +use crate::functions::models::KnownResources; + +/// What is wrong with the `kind` reference to `name`, or `None` when it resolves. +pub(crate) fn reference_problem( + kind: &str, + name: &str, + known: &KnownResources<'_>, +) -> Option { + let function: bool = known.functions.contains(name); + let table_function: bool = known.table_functions.contains(name); + match kind { + REF_KIND if !known.models.contains(name) && known.seeds.contains(name) => Some(format!( + "references seed '{name}' with __ref(...). Use {} for seed references; __ref only \ + resolves models.", + example_call("__seed", name) + )), + REF_KIND if !known.models.contains(name) => { + Some(format!("references unknown model '{name}'")) + } + SEED_KIND if !known.seeds.contains(name) => { + Some(format!("references unknown seed '{name}'")) + } + SOURCE_KIND if !known.sources.contains(name) => { + Some(format!("references unknown source '{name}'")) + } + DBT_REF_KIND => Some(format!( + "uses {} but dbt refs are not supported yet; support may be added in a future release", + example_call("__dbt_ref", name) + )), + UDF_KIND | TABLE_FUNCTION_KIND if !function => { + Some(format!("references unknown SQL function '{name}'")) + } + UDF_KIND if table_function => Some(format!( + "references table function '{name}' with __udf(); use __table_fn()" + )), + TABLE_FUNCTION_KIND if !table_function => Some(format!( + "references scalar function '{name}' with __table_fn(); use __udf() for scalar UDFs" + )), + _ => None, + } +} + +/// `SqlReferenceKind.example_call(name, quote='"')`. +fn example_call(function: &str, name: &str) -> String { + format!("{function}(\"{}\")", name.replace('"', "\"\"")) +} diff --git a/crates/sqlbuild-attachments/src/functions/constants.rs b/crates/sqlbuild-attachments/src/functions/constants.rs new file mode 100644 index 0000000000..f66712c87e --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/constants.rs @@ -0,0 +1,8 @@ +//! Reference kinds a function body's references are validated by. + +pub(crate) const REF_KIND: &str = "ref"; +pub(crate) const SEED_KIND: &str = "seed"; +pub(crate) const SOURCE_KIND: &str = "source"; +pub(crate) const DBT_REF_KIND: &str = "dbt_ref"; +pub(crate) const UDF_KIND: &str = "udf"; +pub(crate) const TABLE_FUNCTION_KIND: &str = "table_fn"; diff --git a/crates/sqlbuild-attachments/src/functions/main/function_reference_error.rs b/crates/sqlbuild-attachments/src/functions/main/function_reference_error.rs new file mode 100644 index 0000000000..ecb5356e0c --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/main/function_reference_error.rs @@ -0,0 +1,19 @@ +//! `validate_function_references`: the error Python raises for a SQL function's bad reference. + +use crate::functions::_helpers::references::reference_problem; +use crate::functions::models::KnownResources; + +/// The error for the first of `references` (`(kind, name)`) the project cannot resolve. +#[must_use] +pub fn function_reference_error( + relative_path: &str, + references: &[(&str, &str)], + known: &KnownResources<'_>, +) -> Option { + for &(kind, name) in references { + if let Some(problem) = reference_problem(kind, name, known) { + return Some(format!("SQL function file {relative_path} {problem}")); + } + } + None +} diff --git a/crates/sqlbuild-attachments/src/functions/main/mod.rs b/crates/sqlbuild-attachments/src/functions/main/mod.rs index 1d35a99aa7..61b48206b1 100644 --- a/crates/sqlbuild-attachments/src/functions/main/mod.rs +++ b/crates/sqlbuild-attachments/src/functions/main/mod.rs @@ -1,2 +1,4 @@ +pub mod function_reference_error; pub mod parse_function_header; pub mod resolve_function_namespace; +pub mod table_function_arity_error; diff --git a/crates/sqlbuild-attachments/src/functions/main/table_function_arity_error.rs b/crates/sqlbuild-attachments/src/functions/main/table_function_arity_error.rs new file mode 100644 index 0000000000..9b4713fa7a --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/main/table_function_arity_error.rs @@ -0,0 +1,33 @@ +//! `validate_table_function_call_arities`: a managed table function called with the wrong count. + +use std::collections::HashMap; + +use crate::functions::constants::TABLE_FUNCTION_KIND; + +/// The first `(kind, name, argument count)` call whose count differs from the declared one. +#[must_use] +pub fn table_function_arity_error( + owner: &str, + calls: &[(&str, &str, Option)], + argument_counts: &HashMap, +) -> Option { + calls.iter().find_map(|&(kind, name, received)| { + if kind != TABLE_FUNCTION_KIND { + return None; + } + let expected: usize = *argument_counts.get(name)?; + if received == Some(expected) { + return None; + } + let label: &str = if expected == 1 { + "argument" + } else { + "arguments" + }; + let received: String = + received.map_or_else(|| "None".to_owned(), |count| count.to_string()); + Some(format!( + "{owner} table function '{name}' expects {expected} {label} but received {received}" + )) + }) +} diff --git a/crates/sqlbuild-attachments/src/functions/mod.rs b/crates/sqlbuild-attachments/src/functions/mod.rs index a2537f3e8e..c279374dac 100644 --- a/crates/sqlbuild-attachments/src/functions/mod.rs +++ b/crates/sqlbuild-attachments/src/functions/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub(crate) mod errors; pub mod main; pub mod models; diff --git a/crates/sqlbuild-attachments/src/functions/models.rs b/crates/sqlbuild-attachments/src/functions/models.rs index 6a0dc12753..95bdbf2ccd 100644 --- a/crates/sqlbuild-attachments/src/functions/models.rs +++ b/crates/sqlbuild-attachments/src/functions/models.rs @@ -95,3 +95,13 @@ pub struct FunctionNamespace { pub fingerprint_logical_database: Option, pub fingerprint_logical_schema: Option, } + +/// The discovered resource names a function's references resolve against. +#[derive(Debug)] +pub struct KnownResources<'a> { + pub models: &'a std::collections::HashSet, + pub seeds: &'a std::collections::HashSet, + pub sources: &'a std::collections::HashSet, + pub functions: &'a std::collections::HashSet, + pub table_functions: &'a std::collections::HashSet, +} diff --git a/crates/sqlbuild-attachments/src/functions/tests.rs b/crates/sqlbuild-attachments/src/functions/tests.rs index 0d422fdc37..785e40079f 100644 --- a/crates/sqlbuild-attachments/src/functions/tests.rs +++ b/crates/sqlbuild-attachments/src/functions/tests.rs @@ -1,8 +1,12 @@ +#[path = "tests/test_function_reference_error.rs"] +mod function_reference_error; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_parse_function_header.rs"] mod parse_function_header; #[path = "tests/test_resolve_function_namespace.rs"] mod resolve_function_namespace; +#[path = "tests/test_table_function_arity_error.rs"] +mod table_function_arity_error; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-attachments/src/functions/tests/helpers.rs b/crates/sqlbuild-attachments/src/functions/tests/helpers.rs index 8ffb3778bc..3a93a6951e 100644 --- a/crates/sqlbuild-attachments/src/functions/tests/helpers.rs +++ b/crates/sqlbuild-attachments/src/functions/tests/helpers.rs @@ -69,3 +69,8 @@ pub(super) fn failure(stage: HeaderStage, message: &str) -> Option std::collections::HashSet { + values.iter().map(|value| (*value).to_owned()).collect() +} diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_function_reference_error.rs b/crates/sqlbuild-attachments/src/functions/tests/test_function_reference_error.rs new file mode 100644 index 0000000000..cbf853f80d --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/tests/test_function_reference_error.rs @@ -0,0 +1,63 @@ +use crate::functions::main::function_reference_error::function_reference_error; +use crate::functions::models::KnownResources; +use crate::functions::tests::helpers::names; +use crate::functions::tests::test_types::FunctionReferenceTestCase; + +#[test] +fn given_function_references_when_validating_then_first_unresolved_one_is_reported() { + let (models, seeds, sources) = (names(&["orders"]), names(&["regions"]), names(&["raw"])); + let (functions, table_functions) = (names(&["score", "open_orders"]), names(&["open_orders"])); + let known = KnownResources { + models: &models, + seeds: &seeds, + sources: &sources, + functions: &functions, + table_functions: &table_functions, + }; + let test_cases = [ + FunctionReferenceTestCase { + description: "every reference resolves", + references: vec![ + ("ref", "orders"), + ("udf", "score"), + ("table_fn", "open_orders"), + ], + expected_error: None, + }, + FunctionReferenceTestCase { + description: "a seed read with __ref names the seed call", + references: vec![("ref", "regions")], + expected_error: Some( + "SQL function file functions/sql/lookup.sql references seed 'regions' with \ + __ref(...). Use __seed(\"regions\") for seed references; __ref only resolves \ + models.", + ), + }, + FunctionReferenceTestCase { + description: "a table function called as a scalar UDF", + references: vec![("ref", "orders"), ("udf", "open_orders")], + expected_error: Some( + "SQL function file functions/sql/lookup.sql references table function \ + 'open_orders' with __udf(); use __table_fn()", + ), + }, + FunctionReferenceTestCase { + description: "a scalar function called as a table function", + references: vec![("table_fn", "score")], + expected_error: Some( + "SQL function file functions/sql/lookup.sql references scalar function 'score' \ + with __table_fn(); use __udf() for scalar UDFs", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + function_reference_error("functions/sql/lookup.sql", &test_case.references, &known) + .as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_table_function_arity_error.rs b/crates/sqlbuild-attachments/src/functions/tests/test_table_function_arity_error.rs new file mode 100644 index 0000000000..f2c60f7267 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/tests/test_table_function_arity_error.rs @@ -0,0 +1,48 @@ +use std::collections::HashMap; + +use crate::functions::main::table_function_arity_error::table_function_arity_error; +use crate::functions::tests::test_types::TableFunctionArityTestCase; + +#[test] +fn given_table_function_calls_when_checking_arity_then_first_mismatch_is_reported() { + let counts: HashMap = HashMap::from([ + ("customer_orders".to_owned(), 1), + ("order_window".to_owned(), 2), + ]); + let test_cases = [ + TableFunctionArityTestCase { + description: "matching counts and other reference kinds pass", + calls: vec![ + ("table_fn", "customer_orders", Some(1)), + ("ref", "orders", None), + ], + expected_error: None, + }, + TableFunctionArityTestCase { + description: "one declared argument reads in the singular", + calls: vec![("table_fn", "customer_orders", Some(2))], + expected_error: Some( + "Model file models/report.sql table function 'customer_orders' expects 1 \ + argument but received 2", + ), + }, + TableFunctionArityTestCase { + description: "an unknown count reads as Python's None", + calls: vec![("table_fn", "order_window", None)], + expected_error: Some( + "Model file models/report.sql table function 'order_window' expects 2 \ + arguments but received None", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + table_function_arity_error("Model file models/report.sql", &test_case.calls, &counts) + .as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_types.rs b/crates/sqlbuild-attachments/src/functions/tests/test_types.rs index 8580b91dc6..4d84c817a7 100644 --- a/crates/sqlbuild-attachments/src/functions/tests/test_types.rs +++ b/crates/sqlbuild-attachments/src/functions/tests/test_types.rs @@ -14,3 +14,15 @@ pub(super) struct FunctionNamespaceTestCase { pub(super) inputs: NamespaceInputs, pub(super) expected_namespace: FunctionNamespace, } + +pub(super) struct FunctionReferenceTestCase { + pub(super) description: &'static str, + pub(super) references: Vec<(&'static str, &'static str)>, + pub(super) expected_error: Option<&'static str>, +} + +pub(super) struct TableFunctionArityTestCase { + pub(super) description: &'static str, + pub(super) calls: Vec<(&'static str, &'static str, Option)>, + pub(super) expected_error: Option<&'static str>, +} diff --git a/crates/sqlbuild-attachments/src/lib.rs b/crates/sqlbuild-attachments/src/lib.rs index cd2db15b49..1ce233f5cd 100644 --- a/crates/sqlbuild-attachments/src/lib.rs +++ b/crates/sqlbuild-attachments/src/lib.rs @@ -6,6 +6,7 @@ pub mod audits; pub mod ceremonial_select; pub mod cursor_intrinsics; pub mod functions; +pub mod reference_markers; pub mod seeds; pub mod sql_lexing; pub mod test_parameters; diff --git a/crates/sqlbuild-attachments/src/reference_markers/constants.rs b/crates/sqlbuild-attachments/src/reference_markers/constants.rs new file mode 100644 index 0000000000..bac1df7be6 --- /dev/null +++ b/crates/sqlbuild-attachments/src/reference_markers/constants.rs @@ -0,0 +1,10 @@ +//! The reference calls that must be resolved before SQL executes, in reporting order. + +/// `ref`, `seed`, `source`, `udf` and `table_fn` call prefixes. +pub(crate) const REFERENCE_CALL_PREFIXES: [&[u8]; 5] = [ + b"__ref(", + b"__seed(", + b"__source(", + b"__udf(", + b"__table_fn(", +]; diff --git a/crates/sqlbuild-attachments/src/reference_markers/main/mod.rs b/crates/sqlbuild-attachments/src/reference_markers/main/mod.rs new file mode 100644 index 0000000000..669871d705 --- /dev/null +++ b/crates/sqlbuild-attachments/src/reference_markers/main/mod.rs @@ -0,0 +1 @@ +pub mod unresolved_reference_marker; diff --git a/crates/sqlbuild-attachments/src/reference_markers/main/unresolved_reference_marker.rs b/crates/sqlbuild-attachments/src/reference_markers/main/unresolved_reference_marker.rs new file mode 100644 index 0000000000..12604ce7e2 --- /dev/null +++ b/crates/sqlbuild-attachments/src/reference_markers/main/unresolved_reference_marker.rs @@ -0,0 +1,33 @@ +//! Python's `assert_no_unresolved_sql_markers`: the first reference call left in executable SQL. + +use crate::reference_markers::constants::REFERENCE_CALL_PREFIXES; +use crate::sql_lexing::main::python_non_code_end::python_non_code_end; +use crate::sql_lexing::main::unclosed_construct::unclosed_construct; +use crate::sql_lexing::models::NonCode; +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +fn marker_in_code(bytes: &[u8], prefix: &[u8]) -> Result { + if !bytes.windows(prefix.len()).any(|window| window == prefix) { + return Ok(false); + } + let mut index: usize = 0; + while index < bytes.len() { + match python_non_code_end(bytes, index) { + NonCode::End(end) => index = end, + NonCode::Raises => return Err(unclosed_construct(bytes, index)), + NonCode::Code if bytes[index..].starts_with(prefix) => return Ok(true), + NonCode::Code => index += 1, + } + } + Ok(false) +} + +/// The index of the first reference kind still called in code, or the construct left unclosed. +pub fn unresolved_reference_marker(sql: &str) -> Result, Unclosed> { + for (position, prefix) in REFERENCE_CALL_PREFIXES.iter().enumerate() { + if marker_in_code(sql.as_bytes(), prefix)? { + return Ok(Some(position)); + } + } + Ok(None) +} diff --git a/crates/sqlbuild-attachments/src/reference_markers/mod.rs b/crates/sqlbuild-attachments/src/reference_markers/mod.rs new file mode 100644 index 0000000000..9b926d1cd2 --- /dev/null +++ b/crates/sqlbuild-attachments/src/reference_markers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-attachments/src/reference_markers/tests.rs b/crates/sqlbuild-attachments/src/reference_markers/tests.rs new file mode 100644 index 0000000000..b867cfafaa --- /dev/null +++ b/crates/sqlbuild-attachments/src/reference_markers/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_unresolved_reference_marker.rs"] +mod unresolved_reference_marker; diff --git a/crates/sqlbuild-attachments/src/reference_markers/tests/test_types.rs b/crates/sqlbuild-attachments/src/reference_markers/tests/test_types.rs new file mode 100644 index 0000000000..3137ec92fd --- /dev/null +++ b/crates/sqlbuild-attachments/src/reference_markers/tests/test_types.rs @@ -0,0 +1,7 @@ +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +pub(super) struct ReferenceMarkerTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_marker: Result, Unclosed>, +} diff --git a/crates/sqlbuild-attachments/src/reference_markers/tests/test_unresolved_reference_marker.rs b/crates/sqlbuild-attachments/src/reference_markers/tests/test_unresolved_reference_marker.rs new file mode 100644 index 0000000000..dcebf38fcd --- /dev/null +++ b/crates/sqlbuild-attachments/src/reference_markers/tests/test_unresolved_reference_marker.rs @@ -0,0 +1,48 @@ +use crate::reference_markers::main::unresolved_reference_marker::unresolved_reference_marker; +use crate::reference_markers::tests::test_types::ReferenceMarkerTestCase; +use sqlbuild_sqltext::sql_scan::models::Unclosed; + +#[test] +fn given_executable_sql_when_scanning_then_only_code_reference_calls_are_reported() { + let test_cases = [ + ReferenceMarkerTestCase { + description: "resolved SQL", + sql: "SELECT * FROM orders", + expected_marker: Ok(None), + }, + ReferenceMarkerTestCase { + description: "references in comments and strings are ignored", + sql: "SELECT '__ref(x)' -- __seed(y)\n/* __source(z) */ FROM $q$__udf()$q$", + expected_marker: Ok(None), + }, + ReferenceMarkerTestCase { + description: "a ref call outranks a later source call", + sql: "SELECT * FROM __source(a) JOIN __ref(b)", + expected_marker: Ok(Some(0)), + }, + ReferenceMarkerTestCase { + description: "a table function call", + sql: "SELECT * FROM __table_fn(\"f\")", + expected_marker: Ok(Some(4)), + }, + ReferenceMarkerTestCase { + description: "an unclosed quote while a marker is present", + sql: "SELECT 'x FROM __ref(a)", + expected_marker: Err(Unclosed::Quote), + }, + ReferenceMarkerTestCase { + description: "an unclosed quote without any marker is not scanned", + sql: "SELECT 'x", + expected_marker: Ok(None), + }, + ]; + + for test_case in test_cases { + assert_eq!( + unresolved_reference_marker(test_case.sql), + test_case.expected_marker, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs new file mode 100644 index 0000000000..dcb580caa4 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs @@ -0,0 +1,272 @@ +//! File operations with Python's `target_writer` semantics. + +use std::collections::HashSet; +use std::fs::{File, OpenOptions}; +use std::io::{ErrorKind, Read, Write}; +use std::path::{Path, PathBuf}; + +use crate::artifacts::constants::NEW_FILE_MODE; +use crate::artifacts::errors::ArtifactError; +use crate::artifacts::models::{Publication, WrittenArtifacts}; + +fn io_error(path: &Path) -> impl FnOnce(std::io::Error) -> ArtifactError + '_ { + move |error| ArtifactError::Io { + path: path.to_path_buf(), + error, + } +} + +pub(crate) fn write_artifacts( + files: &[(PathBuf, Vec)], + check_existing: bool, +) -> Result { + let mut outcome: WrittenArtifacts = WrittenArtifacts::default(); + for (path, contents) in files { + if check_existing && let Some(existing) = read_regular_file(path)? { + if existing == *contents { + outcome.unchanged += 1; + continue; + } + if std::str::from_utf8(&existing).is_err() { + return Err(ArtifactError::ExistingNotUtf8 { path: path.clone() }); + } + } + overwrite_creating_parent(path, contents)?; + outcome.written += 1; + } + Ok(outcome) +} + +/// A regular file's bytes, or `None` when nothing comparable is at `path`. +fn read_regular_file(path: &Path) -> Result>, ArtifactError> { + let mut file: File = match File::open(path) { + Ok(file) => file, + Err(error) if absent(&error) => return Ok(None), + Err(error) => return Err(io_error(path)(error)), + }; + if !file.metadata().map_err(io_error(path))?.is_file() { + return Ok(None); + } + let mut bytes: Vec = Vec::new(); + match file.read_to_end(&mut bytes) { + Ok(_) => Ok(Some(bytes)), + Err(error) if error.kind() == ErrorKind::IsADirectory => Ok(None), + Err(error) => Err(io_error(path)(error)), + } +} + +fn absent(error: &std::io::Error) -> bool { + matches!( + error.kind(), + ErrorKind::NotFound | ErrorKind::NotADirectory | ErrorKind::IsADirectory + ) +} + +fn overwrite_creating_parent(path: &Path, contents: &[u8]) -> Result<(), ArtifactError> { + match overwrite(path, contents) { + Err(error) if error.kind() == ErrorKind::NotFound => { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(io_error(parent))?; + } + overwrite(path, contents).map_err(io_error(path)) + } + result => result.map_err(io_error(path)), + } +} + +fn overwrite(path: &Path, contents: &[u8]) -> std::io::Result<()> { + let mut file: File = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(path)?; + file.write_all(contents) +} + +/// Delete unmanaged files under `compiled_dir`, then every directory left empty. +pub(crate) fn remove_stale( + compiled_dir: &Path, + managed: &HashSet, +) -> Result { + if !compiled_dir.is_dir() { + return Ok(0); + } + let (_, removed) = remove_stale_under(compiled_dir, managed)?; + let _ = std::fs::remove_dir(compiled_dir); + Ok(removed) +} + +/// Clean one directory bottom-up as `os.walk(topdown=False)`; returns (gone, files removed). +fn remove_stale_under( + directory: &Path, + managed: &HashSet, +) -> Result<(bool, usize), ArtifactError> { + let entries = match std::fs::read_dir(directory) { + Ok(entries) => entries, + Err(_) => return Ok((false, 0)), + }; + let mut removed: usize = 0; + let mut kept_file: bool = false; + let mut subdirectories_removed: bool = true; + let mut subdirectories: Vec = Vec::new(); + let mut files: Vec = Vec::new(); + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + if is_directory(&path) { + subdirectories.push(path); + } else { + files.push(path); + } + } + for subdirectory in subdirectories { + if is_link(&subdirectory) { + subdirectories_removed = false; + continue; + } + let (gone, below) = remove_stale_under(&subdirectory, managed)?; + removed += below; + subdirectories_removed &= gone; + } + for file in files { + if managed.contains(&file) { + kept_file = true; + } else { + std::fs::remove_file(&file).map_err(io_error(&file))?; + removed += 1; + } + } + let gone: bool = !kept_file && subdirectories_removed && removed_directory(directory); + Ok((gone, removed)) +} + +fn removed_directory(directory: &Path) -> bool { + match std::fs::remove_dir(directory) { + Ok(()) => true, + Err(_) => false, + } +} + +/// Whether `path` is a directory after following links, as `DirEntry.is_dir()` reports. +fn is_directory(path: &Path) -> bool { + match std::fs::metadata(path) { + Ok(metadata) => metadata.is_dir(), + Err(_) => false, + } +} + +fn is_link(path: &Path) -> bool { + match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata.file_type().is_symlink(), + Err(_) => false, + } +} + +/// Every file under `staged_dir` with its path relative to it, as `os.walk` lists them. +pub(crate) fn staged_files(staged_dir: &Path) -> Vec<(PathBuf, PathBuf)> { + let mut files: Vec<(PathBuf, PathBuf)> = Vec::new(); + let mut pending: Vec = vec![staged_dir.to_path_buf()]; + while let Some(directory) = pending.pop() { + let Ok(entries) = std::fs::read_dir(&directory) else { + continue; + }; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + if !is_directory(&path) { + let relative: PathBuf = path + .strip_prefix(staged_dir) + .map_or_else(|_| path.clone(), Path::to_path_buf); + files.push((path, relative)); + } else if !is_link(&path) { + pending.push(path); + } + } + } + files +} + +pub(crate) fn publish_staged( + staged_dir: &Path, + compiled_dir: &Path, + expected: &HashSet, +) -> Result { + let staged: Vec<(PathBuf, PathBuf)> = staged_files(staged_dir); + let listed: HashSet<&PathBuf> = staged.iter().map(|(_, relative)| relative).collect(); + if listed.len() != expected.len() || !expected.iter().all(|path| listed.contains(path)) { + return Ok(Publication::StagedChanged); + } + if let Some(target_dir) = compiled_dir.parent() { + std::fs::create_dir_all(target_dir).map_err(io_error(target_dir))?; + } + if !compiled_dir.is_dir() + && let Ok(()) = std::fs::rename(staged_dir, compiled_dir) + { + return Ok(Publication::MovedTree); + } + let check_existing: bool = compiled_dir.is_dir(); + let mut published: Vec<(PathBuf, PathBuf)> = Vec::with_capacity(staged.len()); + for (source, relative) in staged { + let path: PathBuf = compiled_dir.join(&relative); + publish_file(&source, &path, check_existing)?; + published.push((source, path)); + } + Ok(Publication::Files(published)) +} + +fn publish_file(source: &Path, path: &Path, check_existing: bool) -> Result<(), ArtifactError> { + if check_existing && path.is_file() { + let contents: Vec = std::fs::read(source).map_err(io_error(source))?; + let existing: Vec = std::fs::read(path).map_err(io_error(path))?; + if existing == contents { + return Ok(()); + } + if std::str::from_utf8(&existing).is_err() { + return Err(ArtifactError::ExistingNotUtf8 { + path: path.to_path_buf(), + }); + } + return overwrite(path, &contents).map_err(io_error(path)); + } + let move_error = |error: std::io::Error| ArtifactError::Move { + source: source.to_path_buf(), + path: path.to_path_buf(), + error, + }; + match move_new_file(source, path) { + Err(error) if error.kind() == ErrorKind::NotFound => { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(io_error(parent))?; + } + move_new_file(source, path).map_err(move_error) + } + result => result.map_err(move_error), + } +} + +fn move_new_file(source: &Path, path: &Path) -> std::io::Result<()> { + match std::fs::rename(source, path) { + Err(error) if error.kind() == ErrorKind::CrossesDevices => copy_new_file(source, path), + result => result, + } +} + +/// Copy across filesystems through a sibling temporary file so the new file appears atomically. +fn copy_new_file(source: &Path, path: &Path) -> std::io::Result<()> { + let directory: &Path = path.parent().unwrap_or_else(|| Path::new(".")); + let mut builder = tempfile::Builder::new(); + let prefix: String = format!( + ".{}.", + path.file_name().unwrap_or_default().to_string_lossy() + ); + let _ = builder.prefix(&prefix).suffix(".tmp"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = builder.permissions(std::fs::Permissions::from_mode(NEW_FILE_MODE)); + } + let mut temporary = builder.tempfile_in(directory)?; + temporary.write_all(&std::fs::read(source)?)?; + temporary + .persist(path) + .map(|_| ()) + .map_err(|error| error.error) +} diff --git a/crates/sqlbuild-cache/src/artifacts/_helpers/mod.rs b/crates/sqlbuild-cache/src/artifacts/_helpers/mod.rs new file mode 100644 index 0000000000..c0f9955931 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod files; diff --git a/crates/sqlbuild-cache/src/artifacts/constants.rs b/crates/sqlbuild-cache/src/artifacts/constants.rs new file mode 100644 index 0000000000..dd298d6931 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/constants.rs @@ -0,0 +1,2 @@ +/// The mode new artifacts are created with before the process umask, as Python's `os.open`. +pub const NEW_FILE_MODE: u32 = 0o666; diff --git a/crates/sqlbuild-cache/src/artifacts/errors.rs b/crates/sqlbuild-cache/src/artifacts/errors.rs new file mode 100644 index 0000000000..425d3e822b --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/errors.rs @@ -0,0 +1,21 @@ +//! Why compiled artifacts could not be written or published. + +use std::path::PathBuf; + +/// An artifact write failed at `path`. +#[derive(Debug)] +pub enum ArtifactError { + /// The operating system refused an operation on `path`. + Io { + path: PathBuf, + error: std::io::Error, + }, + /// The operating system refused to move `source` to `path`. + Move { + source: PathBuf, + path: PathBuf, + error: std::io::Error, + }, + /// The file already at `path` is not UTF-8, which Python reports as a decode error. + ExistingNotUtf8 { path: PathBuf }, +} diff --git a/crates/sqlbuild-cache/src/artifacts/main/mod.rs b/crates/sqlbuild-cache/src/artifacts/main/mod.rs new file mode 100644 index 0000000000..c922563fbf --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/main/mod.rs @@ -0,0 +1,4 @@ +pub mod publish_staged; +pub mod remove_stale; +pub mod staged_files; +pub mod write_artifacts; diff --git a/crates/sqlbuild-cache/src/artifacts/main/publish_staged.rs b/crates/sqlbuild-cache/src/artifacts/main/publish_staged.rs new file mode 100644 index 0000000000..f3f6be3589 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/main/publish_staged.rs @@ -0,0 +1,14 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use crate::artifacts::errors::ArtifactError; +use crate::artifacts::models::Publication; + +/// Publish the expected staged files, moving the whole tree when no compiled directory exists. +pub fn publish_staged( + staged_dir: &Path, + compiled_dir: &Path, + expected: &HashSet, +) -> Result { + crate::artifacts::_helpers::files::publish_staged(staged_dir, compiled_dir, expected) +} diff --git a/crates/sqlbuild-cache/src/artifacts/main/remove_stale.rs b/crates/sqlbuild-cache/src/artifacts/main/remove_stale.rs new file mode 100644 index 0000000000..85cfb929ea --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/main/remove_stale.rs @@ -0,0 +1,12 @@ +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use crate::artifacts::errors::ArtifactError; + +/// Delete files under `compiled_dir` this compile did not manage, then empty directories. +pub fn remove_stale( + compiled_dir: &Path, + managed: &HashSet, +) -> Result { + crate::artifacts::_helpers::files::remove_stale(compiled_dir, managed) +} diff --git a/crates/sqlbuild-cache/src/artifacts/main/staged_files.rs b/crates/sqlbuild-cache/src/artifacts/main/staged_files.rs new file mode 100644 index 0000000000..d829e62232 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/main/staged_files.rs @@ -0,0 +1,9 @@ +use std::path::{Path, PathBuf}; + +/// Every staged file's path relative to `staged_dir`. +pub fn staged_files(staged_dir: &Path) -> Vec { + crate::artifacts::_helpers::files::staged_files(staged_dir) + .into_iter() + .map(|(_, relative)| relative) + .collect() +} diff --git a/crates/sqlbuild-cache/src/artifacts/main/write_artifacts.rs b/crates/sqlbuild-cache/src/artifacts/main/write_artifacts.rs new file mode 100644 index 0000000000..4b44d918a6 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/main/write_artifacts.rs @@ -0,0 +1,12 @@ +use std::path::PathBuf; + +use crate::artifacts::errors::ArtifactError; +use crate::artifacts::models::WrittenArtifacts; + +/// Write each artifact, leaving a byte-identical existing file untouched when asked to check. +pub fn write_artifacts( + files: &[(PathBuf, Vec)], + check_existing: bool, +) -> Result { + crate::artifacts::_helpers::files::write_artifacts(files, check_existing) +} diff --git a/crates/sqlbuild-cache/src/artifacts/mod.rs b/crates/sqlbuild-cache/src/artifacts/mod.rs new file mode 100644 index 0000000000..6b771d183b --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/mod.rs @@ -0,0 +1,7 @@ +//! Compiled artifact files: unchanged-file writes, stale cleanup and staged publication. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-cache/src/artifacts/models.rs b/crates/sqlbuild-cache/src/artifacts/models.rs new file mode 100644 index 0000000000..3e66d6eaca --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/models.rs @@ -0,0 +1,21 @@ +//! What one artifact write or publication did. + +use std::path::PathBuf; + +/// The files a write batch replaced and left unchanged. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct WrittenArtifacts { + pub written: usize, + pub unchanged: usize, +} + +/// How staged artifacts reached the compiled directory. +#[derive(Debug, PartialEq, Eq)] +pub enum Publication { + /// The staged files differ from the expected set; nothing was published. + StagedChanged, + /// The whole staged directory became the compiled directory. + MovedTree, + /// Each staged file was published at its compiled path, in this order. + Files(Vec<(PathBuf, PathBuf)>), +} diff --git a/crates/sqlbuild-cache/src/digest/_helpers/code_files.rs b/crates/sqlbuild-cache/src/digest/_helpers/code_files.rs new file mode 100644 index 0000000000..4da407951a --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/_helpers/code_files.rs @@ -0,0 +1,77 @@ +//! The source files of an editable install, as Python's `os.scandir` walk lists them. + +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::digest::constants::{BYTECODE_CACHE_DIRECTORY, CODE_SOURCE_SUFFIXES}; + +/// Each source file's path below `root`, size and modification time in nanoseconds, sorted. +pub(crate) fn source_file_stats(root: &Path) -> Vec<(String, u64, i128)> { + let mut stats: Vec<(String, u64, i128)> = Vec::new(); + let mut pending: Vec = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + let (directories, files): DirectoryListing = directory_stats(root, &directory); + pending.extend(directories); + stats.extend(files); + } + stats.sort(); + stats +} + +/// A directory's subdirectories to walk and its source files' stats. +type DirectoryListing = (Vec, Vec<(String, u64, i128)>); + +/// One directory's subdirectories and source files; an error ends this directory's listing only. +fn directory_stats(root: &Path, directory: &Path) -> DirectoryListing { + let mut listing: DirectoryListing = (Vec::new(), Vec::new()); + let _ = list_directory(root, directory, &mut listing); + listing +} + +/// Python's `os.scandir` walk of one directory, stopping at its first error. +fn list_directory( + root: &Path, + directory: &Path, + listing: &mut DirectoryListing, +) -> std::io::Result<()> { + let (pending, stats) = listing; + for entry in std::fs::read_dir(directory)? { + let entry: std::fs::DirEntry = entry?; + let path: PathBuf = entry.path(); + if entry.file_type()?.is_dir() { + if entry.file_name() != BYTECODE_CACHE_DIRECTORY { + pending.push(path); + } + continue; + } + let source: bool = Path::new(&entry.file_name()) + .extension() + .and_then(|suffix| suffix.to_str()) + .is_some_and(|suffix| CODE_SOURCE_SUFFIXES.contains(&suffix)); + if !source { + continue; + } + let metadata: std::fs::Metadata = std::fs::metadata(&path)?; + let relative: String = path + .strip_prefix(root) + .unwrap_or(&path) + .to_string_lossy() + .into_owned(); + stats.push((relative, metadata.len(), modified_nanoseconds(&metadata)?)); + } + Ok(()) +} + +/// Python's `st_mtime_ns`: nanoseconds since the epoch, negative before it. +fn modified_nanoseconds(metadata: &std::fs::Metadata) -> std::io::Result { + let modified: SystemTime = metadata.modified()?; + Ok(match modified.duration_since(UNIX_EPOCH) { + Ok(after) => i128::try_from(after.as_nanos()).unwrap_or(i128::MAX), + Err(before) => -i128::try_from(before.duration().as_nanos()).unwrap_or(i128::MAX), + }) +} + +/// Lowercase hexadecimal text of digest bytes, as Python's `hexdigest`. +pub(crate) fn lowercase_hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/crates/sqlbuild-cache/src/digest/_helpers/mod.rs b/crates/sqlbuild-cache/src/digest/_helpers/mod.rs index 208de6bb67..2ebc7ff525 100644 --- a/crates/sqlbuild-cache/src/digest/_helpers/mod.rs +++ b/crates/sqlbuild-cache/src/digest/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod code_files; pub(crate) mod walk; diff --git a/crates/sqlbuild-cache/src/digest/constants.rs b/crates/sqlbuild-cache/src/digest/constants.rs index d0a53f5c3a..8494ec4831 100644 --- a/crates/sqlbuild-cache/src/digest/constants.rs +++ b/crates/sqlbuild-cache/src/digest/constants.rs @@ -30,3 +30,8 @@ pub const PRESENCE_ONLY_FILE_SUFFIXES: &[&str] = &[".duckdb", ".duckdb.wal"]; pub const CONTENT_TAG: &[u8] = b"content"; /// Digest tag of a file that counts only by presence. pub const PRESENCE_TAG: &[u8] = b"presence"; + +/// File suffixes whose sizes and modification times identify an editable install's code. +pub(crate) const CODE_SOURCE_SUFFIXES: &[&str] = &["py", "so", "pyd"]; +/// The bytecode folder an editable install's code walk skips. +pub(crate) const BYTECODE_CACHE_DIRECTORY: &str = "__pycache__"; diff --git a/crates/sqlbuild-cache/src/digest/main/bytes_digest.rs b/crates/sqlbuild-cache/src/digest/main/bytes_digest.rs new file mode 100644 index 0000000000..7b5ec8f290 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/bytes_digest.rs @@ -0,0 +1,8 @@ +//! The BLAKE3 digest of bytes held in memory, equal to `digest_files` for the same contents. + +use crate::digest::types::ContentDigest; + +/// The digest `digest_files` gives a file holding exactly `contents`. +pub fn bytes_digest(contents: &[u8]) -> ContentDigest { + *blake3::hash(contents).as_bytes() +} diff --git a/crates/sqlbuild-cache/src/digest/main/compiled_code_identity.rs b/crates/sqlbuild-cache/src/digest/main/compiled_code_identity.rs new file mode 100644 index 0000000000..6f189b778b --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/compiled_code_identity.rs @@ -0,0 +1,13 @@ +//! The digest that changes with every installed code version and every native rebuild. + +use sha2::{Digest, Sha256}; + +use crate::digest::_helpers::code_files::lowercase_hex; + +/// SHA-256 of the installed code's identity and the native build's identity. +#[must_use] +pub fn compiled_code_identity(installed: &str, build_identity: &str) -> String { + lowercase_hex(&Sha256::digest( + format!("{installed}\0{build_identity}").as_bytes(), + )) +} diff --git a/crates/sqlbuild-cache/src/digest/main/installed_code_identity.rs b/crates/sqlbuild-cache/src/digest/main/installed_code_identity.rs new file mode 100644 index 0000000000..e4ebf39dcd --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/installed_code_identity.rs @@ -0,0 +1,23 @@ +//! The digest that changes with every released or locally edited version of the installed code. + +use std::path::Path; + +use sha2::{Digest, Sha256}; + +use crate::digest::_helpers::code_files::{lowercase_hex, source_file_stats}; + +/// SHA-256 of each fact `\0`-terminated, then each source file below an editable `package_root`. +#[must_use] +pub fn installed_code_identity(facts: &[String], package_root: Option<&Path>) -> String { + let mut hasher: Sha256 = Sha256::new(); + for fact in facts { + hasher.update(fact.as_bytes()); + hasher.update(b"\0"); + } + if let Some(root) = package_root { + for (relative_path, size, modified) in source_file_stats(root) { + hasher.update(format!("{relative_path}\0{size}\0{modified}\0").as_bytes()); + } + } + lowercase_hex(&hasher.finalize()) +} diff --git a/crates/sqlbuild-cache/src/digest/main/mod.rs b/crates/sqlbuild-cache/src/digest/main/mod.rs index c738ef926d..335187a89e 100644 --- a/crates/sqlbuild-cache/src/digest/main/mod.rs +++ b/crates/sqlbuild-cache/src/digest/main/mod.rs @@ -1,4 +1,7 @@ +pub mod bytes_digest; +pub mod compiled_code_identity; pub mod content_digest; pub mod digest_files; pub mod fingerprint_project_files; pub mod hex_digest; +pub mod installed_code_identity; diff --git a/crates/sqlbuild-cache/src/digest/tests.rs b/crates/sqlbuild-cache/src/digest/tests.rs index 7ab98d75a0..329720ef78 100644 --- a/crates/sqlbuild-cache/src/digest/tests.rs +++ b/crates/sqlbuild-cache/src/digest/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_code_identity.rs"] +mod code_identity; #[path = "tests/test_content_digest.rs"] mod content_digest; #[path = "tests/test_digest_files.rs"] diff --git a/crates/sqlbuild-cache/src/digest/tests/helpers.rs b/crates/sqlbuild-cache/src/digest/tests/helpers.rs index 179519088f..e1a0b87918 100644 --- a/crates/sqlbuild-cache/src/digest/tests/helpers.rs +++ b/crates/sqlbuild-cache/src/digest/tests/helpers.rs @@ -27,3 +27,26 @@ pub(crate) fn write_file(root: &Path, relative_path: &str, contents: &str) { pub(crate) fn excluded_files() -> HashSet { HashSet::from([EXCLUDED_MODEL.to_owned()]) } + +/// The modification time Python's oracle gave the file at `index` in sorted path order. +const BASE_MODIFIED_NANOSECONDS: u64 = 1_700_000_000_000_000_000; +const MODIFIED_STEP_NANOSECONDS: u64 = 1_234_567; + +/// Write `files` below `root` with the oracle's modification times and a symlinked folder. +pub(crate) fn write_package(root: &Path, files: &[(&str, &str)]) { + let mut sorted: Vec<&(&str, &str)> = files.iter().collect(); + sorted.sort_by_key(|(path, _)| *path); + for (index, (relative_path, contents)) in sorted.into_iter().enumerate() { + write_file(root, relative_path, contents); + let nanoseconds: u64 = BASE_MODIFIED_NANOSECONDS + + u64::try_from(index).expect("few files") * MODIFIED_STEP_NANOSECONDS; + let modified = std::time::UNIX_EPOCH + std::time::Duration::from_nanos(nanoseconds); + std::fs::File::options() + .write(true) + .open(root.join(relative_path)) + .and_then(|file| file.set_modified(modified)) + .expect("the modification time is set"); + } + #[cfg(unix)] + std::os::unix::fs::symlink("pkg", root.join("link_dir")).expect("the folder link is made"); +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_code_identity.rs b/crates/sqlbuild-cache/src/digest/tests/test_code_identity.rs new file mode 100644 index 0000000000..f4993c3c0a --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/tests/test_code_identity.rs @@ -0,0 +1,74 @@ +use crate::digest::main::compiled_code_identity::compiled_code_identity; +use crate::digest::main::installed_code_identity::installed_code_identity; +use crate::digest::tests::helpers::write_package; +use crate::digest::tests::test_types::{ + CompiledCodeIdentityTestCase, InstalledCodeIdentityTestCase, +}; + +const PACKAGE_FILES: &[(&str, &str)] = &[ + ("a.py", "x"), + ("pkg/b.so", "yy"), + ("pkg/__pycache__/c.py", "z"), + ("pkg/sub/d.pyd", "www"), + (".py", "hidden"), + ("e.txt", "t"), + ("f.tar.py", "ff"), + ("pkg/sub/\u{e9}.py", "u"), +]; +const FACTS: &[&str] = &["1.0", "2.0", "3", "CPython", "3.12"]; + +#[cfg(unix)] +#[test] +fn given_package_tree_when_identifying_installed_code_then_python_digest_is_reproduced() { + let test_cases = [ + InstalledCodeIdentityTestCase { + description: "an editable install hashes sources, skipping bytecode, links and others", + files: PACKAGE_FILES, + facts: FACTS, + editable: true, + expected_identity: "7dba163ff7f85f33293d30ea8a4460b61242b487f65007b171dd7ec8831c22dd", + }, + InstalledCodeIdentityTestCase { + description: "a released install hashes only its facts", + files: PACKAGE_FILES, + facts: FACTS, + editable: false, + expected_identity: "3cd1c8ecd7d241204a1dcebdf9dd0ce4a39c6c64a6403ee6e85a2315075477df", + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("a temporary folder"); + write_package(directory.path(), test_case.files); + let facts: Vec = test_case + .facts + .iter() + .map(|fact| (*fact).to_owned()) + .collect(); + let root = test_case.editable.then_some(directory.path()); + + assert_eq!( + installed_code_identity(&facts, root), + test_case.expected_identity, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_installed_and_build_identities_when_combining_then_python_digest_is_reproduced() { + let test_cases = [CompiledCodeIdentityTestCase { + description: "installed and build identities joined by NUL", + installed: "abc", + build_identity: "build-1", + expected_identity: "b01e2fee4d9cd31dbd7d2ded6d49674d084cf9708e297905c02eb7645ebf015d", + }]; + for test_case in test_cases { + assert_eq!( + compiled_code_identity(test_case.installed, test_case.build_identity), + test_case.expected_identity, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/digest/tests/test_types.rs b/crates/sqlbuild-cache/src/digest/tests/test_types.rs index 3ecb41715d..adae6506da 100644 --- a/crates/sqlbuild-cache/src/digest/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/digest/tests/test_types.rs @@ -33,3 +33,20 @@ pub(crate) struct DigestFilesTestCase { pub(crate) path: fn(&Path) -> PathBuf, pub(crate) expected_digest_of: Option<&'static str>, } + +/// A package tree, the facts hashed before it, and Python's `installed_code_identity` digest. +pub(crate) struct InstalledCodeIdentityTestCase { + pub(crate) description: &'static str, + /// `(relative path, contents)`, given modification times in sorted path order. + pub(crate) files: &'static [(&'static str, &'static str)], + pub(crate) facts: &'static [&'static str], + pub(crate) editable: bool, + pub(crate) expected_identity: &'static str, +} + +pub(crate) struct CompiledCodeIdentityTestCase { + pub(crate) description: &'static str, + pub(crate) installed: &'static str, + pub(crate) build_identity: &'static str, + pub(crate) expected_identity: &'static str, +} diff --git a/crates/sqlbuild-cache/src/lib.rs b/crates/sqlbuild-cache/src/lib.rs index c1e3b017ca..d329f837c8 100644 --- a/crates/sqlbuild-cache/src/lib.rs +++ b/crates/sqlbuild-cache/src/lib.rs @@ -2,5 +2,8 @@ #![forbid(unsafe_code)] +pub mod artifacts; pub mod digest; +pub mod project_reuse; +pub mod project_snapshot; pub mod store; diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs new file mode 100644 index 0000000000..0459e17285 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/attempt.rs @@ -0,0 +1,307 @@ +//! Python's `attempt_reuse` and `write_compile_entry` over the native store. + +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::project_reuse::_helpers::entry::{ + read_entry, remove_entry, rewrite_inputs, write_entry, +}; +use crate::project_reuse::_helpers::files::{ + carried_forward_digests, compare_project_files, needs_refresh, restamped_paths, + stored_project_files, with_missing_digests, +}; +use crate::project_reuse::_helpers::identity::{ + environment_digest, module_stamps, module_stamps_unchanged, search_path_stamps, + settings_inputs_digest, tracked_environment_names, +}; +use crate::project_reuse::_helpers::target::{target_files_unchanged, verified_target_files}; +use crate::project_reuse::_helpers::timings::compile_timings_span; +use crate::project_reuse::errors::ReuseRecordError; +use crate::project_reuse::models::{ + CompileRecord, RecordOutcome, RecordResult, ReuseAttempt, ReuseCheck, ReuseOutcome, ReuseRules, + SettingsInputs, StoredInputs, StoredOutput, +}; +use crate::project_snapshot::main::snapshot_project_files::snapshot_project_files; +use crate::project_snapshot::main::text_path::text_path; +use crate::project_snapshot::models::PathStamp; + +fn now_ns() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_nanos()).unwrap_or(i64::MAX) + }) +} + +fn bypassed(check: &ReuseCheck<'_>) -> bool { + check.bypass_requested + || check.rules.disabling_variables.iter().any(|(name, value)| { + std::env::var_os(name).is_some_and(|current| current == value.as_str()) + }) + || !check + .rules + .config_filenames + .iter() + .any(|name| check.project_dir.join(name).is_file()) +} + +fn snapshot(project_dir: &Path, rules: &ReuseRules) -> Vec { + snapshot_project_files(project_dir, &rules.snapshot) +} + +pub(crate) fn check(check: ReuseCheck<'_>) -> ReuseAttempt { + let rules: &ReuseRules = check.rules; + let mut attempt: ReuseAttempt = ReuseAttempt { + outcome: ReuseOutcome::Bypass, + project_dir: check.project_dir.clone(), + store_path: check.store_path.clone(), + identity: check.identity.clone(), + search_path: Vec::new(), + snapshot: Vec::new(), + snapshot_ns: 0, + digests: HashMap::new(), + restamped: HashSet::new(), + replay: None, + replay_failed: false, + digested_files: 0, + }; + if bypassed(&check) { + return attempt; + } + attempt.outcome = ReuseOutcome::Miss; + attempt.snapshot_ns = now_ns(); + attempt.search_path = + search_path_stamps(&check.identity.search_path, &check.project_dir, rules); + attempt.snapshot = snapshot(&check.project_dir, rules); + if let Some((inputs, output)) = read_entry(&check.store_path) { + let comparison: Comparison = checked(&attempt, &inputs, rules); + attempt.digested_files += comparison.digested_files; + attempt.digests = comparison.digests; + if comparison.unchanged { + if check.json_output && output.timings_span.is_none() { + attempt.replay_failed = true; + return attempt; + } + attempt.outcome = ReuseOutcome::Hit; + if needs_refresh(&inputs.project_files, &attempt.snapshot) { + let refreshed: StoredInputs = StoredInputs { + project_files: stored_project_files( + &attempt.snapshot, + &attempt.digests, + attempt.snapshot_ns, + rules.racy_window_ns, + ), + ..inputs + }; + rewrite_inputs(&check.store_path, &refreshed); + } + attempt.replay = Some(output); + return attempt; + } + attempt.restamped = restamped_paths(&inputs.project_files, &attempt.snapshot); + } + let known: usize = attempt.digests.len(); + attempt.digests = with_missing_digests(&attempt, &HashSet::new(), rules.racy_window_ns); + attempt.digested_files += attempt.digests.len() - known; + attempt +} + +/// Whether a stored compile still matches, with the digests still valid either way. +struct Comparison { + unchanged: bool, + digests: HashMap, + digested_files: usize, +} + +/// Compare every stored identity, carrying forward digests still valid either way. +fn checked(attempt: &ReuseAttempt, inputs: &StoredInputs, rules: &ReuseRules) -> Comparison { + let identity_unchanged: bool = inputs.invocation_digest == attempt.identity.invocation_digest + && inputs.runtime == attempt.identity.runtime + && inputs.search_path == attempt.search_path + && tracked_environment_names(&inputs.environment_names, rules) == inputs.environment_names + && environment_digest(&inputs.environment_names, rules) == inputs.environment_digest + && settings_inputs_digest(&inputs.settings_inputs, rules) == inputs.settings_digest + && module_stamps_unchanged(&inputs.modules); + let (files_unchanged, verified, digested_files) = if identity_unchanged { + compare_project_files( + &attempt.project_dir, + &inputs.project_files, + &attempt.snapshot, + ) + } else { + (false, HashMap::new(), 0) + }; + Comparison { + unchanged: files_unchanged + && target_files_unchanged( + &inputs.target_files, + &attempt.project_dir, + inputs.target_tree, + rules, + ), + digests: carried_forward_digests(&inputs.project_files, &attempt.snapshot, verified), + digested_files, + } +} + +pub(crate) fn record( + attempt: &ReuseAttempt, + record: &CompileRecord, + rules: &ReuseRules, +) -> Result { + if attempt.outcome != ReuseOutcome::Miss { + return Ok(RecordResult { + outcome: RecordOutcome::NotAttempted, + digested_files: 0, + }); + } + let result: Result = store(attempt, record, rules); + if !matches!( + result, + Ok(RecordResult { + outcome: RecordOutcome::Stored, + .. + }) + ) { + remove_entry(&attempt.store_path); + } + remove_retired(&attempt.store_path, rules); + result +} + +fn store( + attempt: &ReuseAttempt, + record: &CompileRecord, + rules: &ReuseRules, +) -> Result { + let removed: RecordResult = RecordResult { + outcome: RecordOutcome::Removed, + digested_files: 0, + }; + let (Some(stdout), Ok(settings_inputs)) = (&record.stdout, &record.settings) else { + return Ok(removed); + }; + if !record.compile_cache_enabled || record.read_run_id { + return Ok(removed); + } + let project_dir: &Path = &attempt.project_dir; + let digests: HashMap = + with_missing_digests(attempt, &attempt.restamped, rules.racy_window_ns); + let digested_files: usize = digests.len() - attempt.digests.len(); + let unchanged_project: bool = same_snapshot(&snapshot(project_dir, rules), &attempt.snapshot); + let target_files: Option> = if unchanged_project { + verified_target_files(project_dir, record, attempt.snapshot_ns, rules) + } else { + None + }; + let Some(target_files) = target_files else { + return Ok(RecordResult { + outcome: RecordOutcome::Skipped, + digested_files, + }); + }; + let recorded: Recorded<'_> = Recorded { + settings_inputs, + digests: &digests, + target_files, + }; + let inputs: StoredInputs = stored_inputs(attempt, record, rules, recorded); + let output: StoredOutput = StoredOutput { + stderr_lines: record.stderr_lines.clone(), + exit_code: record.exit_code, + timings_span: if record.json_output { + compile_timings_span(stdout) + } else { + None + }, + stdout: stdout.clone(), + }; + if stdout.len() > rules.max_entry_bytes { + return Ok(RecordResult { + outcome: RecordOutcome::Removed, + digested_files, + }); + } + write_entry( + &attempt.store_path, + &inputs, + &output, + rules.max_stored_entries, + )?; + Ok(RecordResult { + outcome: RecordOutcome::Stored, + digested_files, + }) +} + +/// What recording measured for one stored compile. +struct Recorded<'a> { + settings_inputs: &'a [SettingsInputs], + digests: &'a HashMap, + target_files: Vec, +} + +fn stored_inputs( + attempt: &ReuseAttempt, + record: &CompileRecord, + rules: &ReuseRules, + recorded: Recorded<'_>, +) -> StoredInputs { + let Recorded { + settings_inputs, + digests, + target_files, + } = recorded; + let environment_names: Vec = + tracked_environment_names(&record.template_environment_names, rules); + let covered: HashSet = attempt + .snapshot + .iter() + .map(|stamp| { + attempt + .project_dir + .join(text_path(&stamp.relative_path)) + .to_string_lossy() + .into_owned() + }) + .collect(); + StoredInputs { + invocation_digest: attempt.identity.invocation_digest.clone(), + runtime: attempt.identity.runtime.clone(), + search_path: attempt.search_path.clone(), + environment_digest: environment_digest(&environment_names, rules), + environment_names, + modules: module_stamps(&record.module_paths, &covered), + project_files: stored_project_files( + &attempt.snapshot, + digests, + attempt.snapshot_ns, + rules.racy_window_ns, + ), + target_files, + target_tree: record.artifacts_written, + settings_digest: settings_inputs_digest(settings_inputs, rules), + settings_inputs: settings_inputs.to_vec(), + } +} + +/// Delete stored compiles older releases left in their own directories. +fn remove_retired(store_path: &Path, rules: &ReuseRules) { + let Some(cache_directory) = store_path.parent().and_then(Path::parent) else { + return; + }; + for name in &rules.retired_directories { + let _ = std::fs::remove_dir_all(cache_directory.join(name)); + } +} + +fn same_snapshot(left: &[PathStamp], right: &[PathStamp]) -> bool { + let index = |stamps: &[PathStamp]| -> HashMap { + stamps + .iter() + .map(|stamp| (stamp.relative_path.clone(), stamp.clone())) + .collect() + }; + left.len() == right.len() && index(left) == index(right) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs new file mode 100644 index 0000000000..e3603b78d5 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/entry.rs @@ -0,0 +1,347 @@ +//! One stored compile per selected target, kept as one shared native store file. + +use std::path::{Path, PathBuf}; + +use crate::digest::main::content_digest::content_digest; +use crate::digest::types::ContentDigest; +use crate::project_reuse::constants::{ + REUSE_STORE_ENVIRONMENT, REUSE_STORE_KIND, REUSE_STORE_SUFFIX, +}; +use crate::project_reuse::models::{SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile}; +use crate::project_snapshot::models::PathStamp; +use crate::store::errors::StoreDecodeError; +use crate::store::main::open_native_store::open_native_store; +use crate::store::models::{NativeStore, RecordReader, RecordWriter}; + +const KINDS: [&str; 7] = ["f", "fl", "d", "dl", "bl", "s", "p"]; + +fn inputs_key() -> ContentDigest { + content_digest(&["inputs"]) +} + +fn output_key() -> ContentDigest { + content_digest(&["output"]) +} + +fn stdout_key(stdout: &str) -> ContentDigest { + content_digest(&["stdout", stdout]) +} + +/// The stored inputs and output, or `None` when the slot is empty, damaged or foreign. +pub(crate) fn read_entry(path: &Path) -> Option<(StoredInputs, StoredOutput)> { + let mut store: NativeStore = + match open_native_store(path, REUSE_STORE_KIND, REUSE_STORE_ENVIRONMENT) { + Ok(store) => store, + Err(_) => return None, + }; + let Ok(inputs) = decode_inputs(store.get(&inputs_key())?) else { + return None; + }; + let Ok((mut output, digest)) = decode_output(store.get(&output_key())?) else { + return None; + }; + let Ok(stdout) = String::from_utf8(store.get(&digest)?.to_vec()) else { + return None; + }; + output.stdout = stdout; + if stdout_key(&output.stdout) != digest { + return None; + } + Some((inputs, output)) +} + +/// Replace the slot with one stored compile, then keep at most `max_entries` slots. +pub(crate) fn write_entry( + path: &Path, + inputs: &StoredInputs, + output: &StoredOutput, + max_entries: usize, +) -> std::io::Result<()> { + let mut store: NativeStore = NativeStore { + kind: REUSE_STORE_KIND.to_owned(), + environment: REUSE_STORE_ENVIRONMENT.to_owned(), + ..NativeStore::default() + }; + let digest: ContentDigest = stdout_key(&output.stdout); + store.put(inputs_key(), encode_inputs(inputs)); + store.put(output_key(), encode_output(output, &digest)); + store.put(digest, output.stdout.as_bytes().to_vec()); + let _ = store.save(path, &[])?; + prune_entries(path, max_entries); + Ok(()) +} + +/// Replace only the stored inputs, keeping the stored output. +pub(crate) fn rewrite_inputs(path: &Path, inputs: &StoredInputs) { + if let Some((_, output)) = read_entry(path) { + let _ = write_entry(path, inputs, &output, usize::MAX); + } +} + +pub(crate) fn remove_entry(path: &Path) { + let _ = std::fs::remove_file(path); +} + +fn prune_entries(keep: &Path, max_entries: usize) { + let Some(directory) = keep.parent() else { + return; + }; + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + let mut stored: Vec<(std::time::SystemTime, PathBuf)> = entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| path != keep) + .filter(|path| { + path.extension() + .is_some_and(|suffix| suffix == REUSE_STORE_SUFFIX) + }) + .filter_map(|path| { + let modified = std::fs::metadata(&path).and_then(|metadata| metadata.modified()); + match modified { + Ok(modified) => Some((modified, path)), + Err(_) => None, + } + }) + .collect(); + stored.sort_by(|left, right| right.cmp(left)); + for (_, stale) in stored.into_iter().skip(max_entries.saturating_sub(1)) { + remove_entry(&stale); + } +} + +/// The record fields stored compiles use beyond the shared writer's primitives. +trait ReuseRecordWriter { + fn put_strings(&mut self, values: &[String]); + fn put_i64(&mut self, value: i64); + fn put_optional(&mut self, value: Option<&str>); + fn put_stamp(&mut self, stamp: &PathStamp); +} + +impl ReuseRecordWriter for RecordWriter { + fn put_strings(&mut self, values: &[String]) { + self.put_u64(values.len() as u64); + for value in values { + self.put_str(value); + } + } + + fn put_i64(&mut self, value: i64) { + self.put_u64(value as u64); + } + + fn put_optional(&mut self, value: Option<&str>) { + match value { + Some(value) => { + self.put_u64(1); + self.put_str(value); + } + None => self.put_u64(0), + } + } + + fn put_stamp(&mut self, stamp: &PathStamp) { + self.put_str(&stamp.relative_path); + self.put_str(stamp.kind); + self.put_u64(stamp.size); + self.put_i64(stamp.mtime_ns); + self.put_i64(stamp.ctime_ns); + self.put_u64(stamp.inode); + self.put_optional(stamp.link.as_deref()); + } +} + +/// The record fields stored compiles read beyond the shared reader's primitives. +trait ReuseRecordReader { + fn strings(&mut self) -> Result, StoreDecodeError>; + fn read_i64(&mut self) -> Result; + fn optional(&mut self) -> Result, StoreDecodeError>; + fn stamp(&mut self) -> Result; + fn flag(&mut self) -> Result; +} + +impl ReuseRecordReader for RecordReader<'_> { + fn strings(&mut self) -> Result, StoreDecodeError> { + let count: u64 = self.u64()?; + (0..count).map(|_| self.string()).collect() + } + + fn read_i64(&mut self) -> Result { + Ok(self.u64()? as i64) + } + + fn optional(&mut self) -> Result, StoreDecodeError> { + match self.u64()? { + 0 => Ok(None), + 1 => self.string().map(Some), + _ => Err(StoreDecodeError), + } + } + + fn stamp(&mut self) -> Result { + let relative_path: String = self.string()?; + let kind_text: String = self.string()?; + let kind: &'static str = KINDS + .iter() + .find(|kind| **kind == kind_text) + .copied() + .ok_or(StoreDecodeError)?; + Ok(PathStamp { + relative_path, + kind, + size: self.u64()?, + mtime_ns: self.read_i64()?, + ctime_ns: self.read_i64()?, + inode: self.u64()?, + link: self.optional()?, + }) + } + + fn flag(&mut self) -> Result { + match self.u64()? { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(StoreDecodeError), + } + } +} + +fn encode_inputs(inputs: &StoredInputs) -> Vec { + let mut writer: RecordWriter = RecordWriter::default(); + writer.put_str(&inputs.invocation_digest); + writer.put_u64(inputs.runtime.len() as u64); + for (name, value) in &inputs.runtime { + writer.put_str(name); + writer.put_str(value); + } + writer.put_u64(inputs.search_path.len() as u64); + for (entry, mtime_ns) in &inputs.search_path { + writer.put_str(entry); + writer.put_i64(*mtime_ns); + } + writer.put_strings(&inputs.environment_names); + writer.put_str(&inputs.environment_digest); + writer.put_u64(inputs.modules.len() as u64); + for (path, mtime_ns, size) in &inputs.modules { + writer.put_str(path); + writer.put_i64(*mtime_ns); + writer.put_u64(*size); + } + writer.put_u64(inputs.project_files.len() as u64); + for file in &inputs.project_files { + writer.put_stamp(&file.stamp); + writer.put_optional(file.digest.as_deref()); + writer.put_u64(u64::from(file.racy)); + } + writer.put_u64(inputs.target_files.len() as u64); + for target in &inputs.target_files { + writer.put_stamp(target); + } + writer.put_u64(u64::from(inputs.target_tree)); + writer.put_u64(inputs.settings_inputs.len() as u64); + for settings in &inputs.settings_inputs { + writer.put_u64(u64::from(settings.case_sensitive)); + writer.put_strings(&settings.names); + writer.put_strings(&settings.prefixes); + writer.put_strings(&settings.env_files); + writer.put_strings(&settings.secrets_dirs); + } + writer.put_str(&inputs.settings_digest); + writer.into_bytes() +} + +fn decode_inputs(bytes: &[u8]) -> Result { + let mut reader: RecordReader<'_> = RecordReader::new(bytes); + let invocation_digest: String = reader.string()?; + let runtime = (0..reader.u64()?) + .map(|_| Ok((reader.string()?, reader.string()?))) + .collect::>()?; + let search_path = (0..reader.u64()?) + .map(|_| Ok((reader.string()?, reader.read_i64()?))) + .collect::>()?; + let environment_names: Vec = reader.strings()?; + let environment_digest: String = reader.string()?; + let modules = (0..reader.u64()?) + .map(|_| Ok((reader.string()?, reader.read_i64()?, reader.u64()?))) + .collect::>()?; + let project_files = (0..reader.u64()?) + .map(|_| { + Ok(StoredProjectFile { + stamp: reader.stamp()?, + digest: reader.optional()?, + racy: reader.flag()?, + }) + }) + .collect::>()?; + let target_files = (0..reader.u64()?) + .map(|_| reader.stamp()) + .collect::>()?; + let target_tree: bool = reader.flag()?; + let settings_inputs = (0..reader.u64()?) + .map(|_| { + Ok(SettingsInputs { + case_sensitive: reader.flag()?, + names: reader.strings()?, + prefixes: reader.strings()?, + env_files: reader.strings()?, + secrets_dirs: reader.strings()?, + }) + }) + .collect::>()?; + let settings_digest: String = reader.string()?; + reader.finish()?; + Ok(StoredInputs { + invocation_digest, + runtime, + search_path, + environment_names, + environment_digest, + modules, + project_files, + target_files, + target_tree, + settings_inputs, + settings_digest, + }) +} + +fn encode_output(output: &StoredOutput, stdout: &ContentDigest) -> Vec { + let mut writer: RecordWriter = RecordWriter::default(); + writer.put_strings(&output.stderr_lines); + writer.put_i64(output.exit_code); + match output.timings_span { + Some((start, end)) => { + writer.put_u64(1); + writer.put_u64(start as u64); + writer.put_u64(end as u64); + } + None => writer.put_u64(0), + } + writer.put_bytes(stdout); + writer.into_bytes() +} + +fn decode_output(bytes: &[u8]) -> Result<(StoredOutput, ContentDigest), StoreDecodeError> { + let mut reader: RecordReader<'_> = RecordReader::new(bytes); + let stderr_lines: Vec = reader.strings()?; + let exit_code: i64 = reader.read_i64()?; + let timings_span: Option<(usize, usize)> = if reader.flag()? { + let start: usize = usize::try_from(reader.u64()?).map_err(|_| StoreDecodeError)?; + let end: usize = usize::try_from(reader.u64()?).map_err(|_| StoreDecodeError)?; + Some((start, end)) + } else { + None + }; + let digest: ContentDigest = reader.bytes()?.try_into().map_err(|_| StoreDecodeError)?; + reader.finish()?; + Ok(( + StoredOutput { + stderr_lines, + exit_code, + timings_span, + stdout: String::new(), + }, + digest, + )) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs new file mode 100644 index 0000000000..69c98dbfb4 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/files.rs @@ -0,0 +1,204 @@ +//! Project file comparison and content digests, as Python's compile reuse computes them. + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +use crate::digest::main::digest_files::digest_files; +use crate::digest::main::hex_digest::hex_digest; +use crate::project_reuse::constants::NATIVE_DIGEST_PREFIX; +use crate::project_reuse::models::{ReuseAttempt, StoredProjectFile}; +use crate::project_snapshot::main::text_path::text_path; +use crate::project_snapshot::models::PathStamp; + +const FILE: &str = "f"; +const FILE_LINK: &str = "fl"; +const PRESENCE_ONLY: [&str; 4] = ["d", "dl", "bl", "p"]; + +pub(crate) fn hashable(stamp: &PathStamp) -> bool { + stamp.kind == FILE || stamp.kind == FILE_LINK +} + +fn presence_only(stamp: &PathStamp) -> bool { + PRESENCE_ONLY.contains(&stamp.kind) +} + +/// Each file's native content digest, or `None` where it cannot be read. +pub(crate) fn file_digests(paths: &[PathBuf]) -> Vec> { + digest_files(paths) + .into_iter() + .map(|digest| match digest { + Ok(digest) => Some(format!("{NATIVE_DIGEST_PREFIX}{}", hex_digest(&digest))), + Err(_) => None, + }) + .collect() +} + +pub(crate) fn file_digest(path: &Path) -> Option { + file_digests(&[path.to_path_buf()]).pop().flatten() +} + +pub(crate) fn is_racy(stamp: &PathStamp, snapshot_ns: i64, window_ns: i64) -> bool { + hashable(stamp) && stamp.mtime_ns.max(stamp.ctime_ns) >= snapshot_ns - window_ns +} + +fn by_path(stored: &[StoredProjectFile]) -> HashMap<&str, &StoredProjectFile> { + stored + .iter() + .map(|file| (file.stamp.relative_path.as_str(), file)) + .collect() +} + +fn same_identity(stamp: &PathStamp, recorded: &PathStamp) -> bool { + stamp.kind == recorded.kind + && stamp.link == recorded.link + && (presence_only(stamp) || stamp.size == recorded.size) +} + +/// Whether every project path is unchanged, hashing only where stat moved or may be racy. +pub(crate) fn compare_project_files( + project_dir: &Path, + stored: &[StoredProjectFile], + current: &[PathStamp], +) -> (bool, HashMap, usize) { + let mut verified: HashMap = HashMap::new(); + let mut read: usize = 0; + if stored.len() != current.len() { + return (false, verified, read); + } + let previous_by_path: HashMap<&str, &StoredProjectFile> = by_path(stored); + for stamp in current { + let Some(previous) = previous_by_path.get(stamp.relative_path.as_str()) else { + return (false, verified, read); + }; + if !same_identity(stamp, &previous.stamp) { + return (false, verified, read); + } + if presence_only(stamp) || (*stamp == previous.stamp && !previous.racy) { + continue; + } + let Some(previous_digest) = previous.digest.as_ref().filter(|_| hashable(stamp)) else { + return (false, verified, read); + }; + read += 1; + match file_digest(&project_dir.join(text_path(&stamp.relative_path))) { + Some(digest) if digest == *previous_digest => { + let _ = verified.insert(stamp.relative_path.clone(), digest); + } + _ => return (false, verified, read), + } + } + (true, verified, read) +} + +/// Digests still valid for current files, without reading any content. +pub(crate) fn carried_forward_digests( + stored: &[StoredProjectFile], + current: &[PathStamp], + verified: HashMap, +) -> HashMap { + let previous_by_path: HashMap<&str, &StoredProjectFile> = by_path(stored); + let mut digests: HashMap = verified; + for stamp in current { + if digests.contains_key(&stamp.relative_path) { + continue; + } + if let Some(previous) = previous_by_path.get(stamp.relative_path.as_str()) + && let Some(digest) = &previous.digest + && !previous.racy + && previous.stamp == *stamp + { + let _ = digests.insert(stamp.relative_path.clone(), digest.clone()); + } + } + digests +} + +/// Hashable files whose stat identity moved since the stored compile. +pub(crate) fn restamped_paths( + stored: &[StoredProjectFile], + current: &[PathStamp], +) -> HashSet { + let previous_by_path: HashMap<&str, &StoredProjectFile> = by_path(stored); + current + .iter() + .filter(|stamp| { + hashable(stamp) + && previous_by_path + .get(stamp.relative_path.as_str()) + .is_some_and(|previous| { + previous.stamp.kind == stamp.kind && previous.stamp != **stamp + }) + }) + .map(|stamp| stamp.relative_path.clone()) + .collect() +} + +/// Whether a hit verified any file by content, so its new stamp is worth storing. +pub(crate) fn needs_refresh(stored: &[StoredProjectFile], current: &[PathStamp]) -> bool { + let previous_by_path: HashMap<&str, &StoredProjectFile> = by_path(stored); + current.iter().any(|stamp| { + previous_by_path + .get(stamp.relative_path.as_str()) + .is_some_and(|previous| (previous.racy || previous.stamp != *stamp) && hashable(stamp)) + }) +} + +/// Each project path's stat identity with any digest already known for it. +pub(crate) fn stored_project_files( + snapshot: &[PathStamp], + digests: &HashMap, + snapshot_ns: i64, + window_ns: i64, +) -> Vec { + snapshot + .iter() + .map(|stamp| StoredProjectFile { + stamp: stamp.clone(), + digest: digests.get(&stamp.relative_path).cloned(), + racy: is_racy(stamp, snapshot_ns, window_ns), + }) + .collect() +} + +/// The attempt's digests extended with racy files, plus `paths`, that still lack one. +pub(crate) fn with_missing_digests( + attempt: &ReuseAttempt, + paths: &HashSet, + window_ns: i64, +) -> HashMap { + let mut completed: HashMap = attempt.digests.clone(); + let missing: Vec<&PathStamp> = attempt + .snapshot + .iter() + .filter(|stamp| { + hashable(stamp) + && !completed.contains_key(&stamp.relative_path) + && (paths.contains(&stamp.relative_path) + || is_racy(stamp, attempt.snapshot_ns, window_ns)) + }) + .collect(); + let files: Vec = missing + .iter() + .map(|stamp| attempt.project_dir.join(text_path(&stamp.relative_path))) + .collect(); + for (stamp, digest) in missing.iter().zip(file_digests(&files)) { + if let Some(digest) = digest { + let _ = completed.insert(stamp.relative_path.clone(), digest); + } + } + completed +} + +/// Bytes hashing `paths` that still lack a digest would read. +pub(crate) fn pending_digest_bytes( + snapshot: &[PathStamp], + digests: &HashMap, + paths: &HashSet, +) -> u64 { + snapshot + .iter() + .filter(|stamp| paths.contains(&stamp.relative_path)) + .filter(|stamp| !digests.contains_key(&stamp.relative_path)) + .map(|stamp| stamp.size) + .sum() +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/identity.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/identity.rs new file mode 100644 index 0000000000..e3043bbc1c --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/identity.rs @@ -0,0 +1,237 @@ +//! Search path, environment, provider settings and module identity of a compile. + +use std::collections::HashSet; +use std::ffi::OsString; +use std::path::{Path, PathBuf}; + +use crate::project_reuse::_helpers::files::file_digest; +use crate::project_reuse::constants::NATIVE_DIGEST_PREFIX; +use crate::project_reuse::models::{ReuseRules, SettingsInputs}; +use crate::project_snapshot::main::text_path::text_path; + +/// Stat each `sys.path` entry, leaving the project root to the project walk. +pub(crate) fn search_path_stamps( + search_path: &[String], + project_dir: &Path, + rules: &ReuseRules, +) -> Vec<(String, i64)> { + let project_root: Option = resolved(project_dir); + search_path + .iter() + .map(|entry| { + let path: PathBuf = if entry.is_empty() { + PathBuf::from(".") + } else { + text_path(entry) + }; + let mtime_ns: i64 = if project_root.is_some() && resolved(&path) == project_root { + rules.project_root_path_mtime_ns + } else { + match std::fs::metadata(&path) { + Ok(metadata) => modified_ns(&metadata), + Err(_) => rules.missing_path_mtime_ns, + } + }; + (entry.clone(), mtime_ns) + }) + .collect() +} + +fn resolved(path: &Path) -> Option { + let Ok(resolved) = std::fs::canonicalize(path) else { + return None; + }; + Some(resolved) +} + +#[cfg(unix)] +fn modified_ns(metadata: &std::fs::Metadata) -> i64 { + use std::os::unix::fs::MetadataExt; + metadata.mtime() * 1_000_000_000 + metadata.mtime_nsec() +} + +#[cfg(not(unix))] +fn modified_ns(metadata: &std::fs::Metadata) -> i64 { + let Ok(modified) = metadata.modified() else { + return 0; + }; + match modified.duration_since(std::time::UNIX_EPOCH) { + Ok(elapsed) => i64::try_from(elapsed.as_nanos()).unwrap_or(i64::MAX), + Err(_) => 0, + } +} + +#[cfg(unix)] +fn os_bytes(value: &OsString) -> Vec { + use std::os::unix::ffi::OsStrExt; + value.as_bytes().to_vec() +} + +#[cfg(not(unix))] +fn os_bytes(value: &OsString) -> Vec { + value.to_string_lossy().into_owned().into_bytes() +} + +/// Tracked SQLBuild variables (not engine or capture settings) plus template reads, sorted. +pub(crate) fn tracked_environment_names( + template_names: &[String], + rules: &ReuseRules, +) -> Vec { + let mut names: Vec = std::env::vars_os() + .filter_map(|(name, _)| unicode_name(name)) + .filter(|name| tracked(name, rules)) + .chain(template_names.iter().cloned()) + .collect::>() + .into_iter() + .collect(); + names.sort(); + names +} + +fn unicode_name(name: OsString) -> Option { + let Ok(name) = name.into_string() else { + return None; + }; + Some(name) +} + +fn tracked(name: &str, rules: &ReuseRules) -> bool { + has_prefix(name, &rules.tracked_prefixes) + && !rules + .untracked_names + .iter() + .any(|untracked| untracked == name) +} + +fn has_prefix(name: &str, prefixes: &[String]) -> bool { + prefixes + .iter() + .any(|prefix| name.starts_with(prefix.as_str())) +} + +/// Digest the current values of `names` without retaining them. +pub(crate) fn environment_digest(names: &[String], rules: &ReuseRules) -> String { + let mut hasher = blake3::Hasher::new(); + for name in names { + let _ = hasher.update(name.as_bytes()); + let _ = hasher.update(b"\0"); + let value: Vec = std::env::var_os(name).map_or_else( + || rules.missing_environment_value.as_bytes().to_vec(), + |value| os_bytes(&value), + ); + let _ = hasher.update(&value); + let _ = hasher.update(b"\0"); + } + format!("{NATIVE_DIGEST_PREFIX}{}", hasher.finalize().to_hex()) +} + +/// Digest the provider settings values in the environment, env files and secrets directories. +pub(crate) fn settings_inputs_digest(inputs: &[SettingsInputs], rules: &ReuseRules) -> String { + let mut fields: Vec> = Vec::new(); + for item in inputs { + for (name, value) in settings_environment(item) { + fields.extend([b"env".to_vec(), name, value]); + } + for path in &item.env_files { + fields.extend([ + b"env_file".to_vec(), + path.as_bytes().to_vec(), + contents_digest(Path::new(path), rules).into_bytes(), + ]); + } + for directory in &item.secrets_dirs { + fields.extend([b"secrets_dir".to_vec(), directory.as_bytes().to_vec()]); + for (relative, digest) in directory_contents(Path::new(directory), rules) { + fields.extend([relative.into_bytes(), digest.into_bytes()]); + } + } + } + let encoded: Vec = fields.join(&b"\0"[..]); + format!("{NATIVE_DIGEST_PREFIX}{}", blake3::hash(&encoded).to_hex()) +} + +fn settings_environment(inputs: &SettingsInputs) -> Vec<(Vec, Vec)> { + let mut matched: Vec<(Vec, Vec)> = std::env::vars_os() + .filter(|(name, _)| { + let name: String = name.to_string_lossy().into_owned(); + let key: String = if inputs.case_sensitive { + name + } else { + name.to_lowercase() + }; + inputs.names.contains(&key) || has_prefix(&key, &inputs.prefixes) + }) + .map(|(name, value)| (os_bytes(&name), os_bytes(&value))) + .collect(); + matched.sort(); + matched +} + +fn contents_digest(path: &Path, rules: &ReuseRules) -> String { + file_digest(path).unwrap_or_else(|| rules.missing_file_digest.clone()) +} + +/// Every file under `directory`, top-down with sorted names, as `os.walk` lists them. +fn directory_contents(directory: &Path, rules: &ReuseRules) -> Vec<(String, String)> { + let mut contents: Vec<(String, String)> = Vec::new(); + let mut pending: Vec = vec![directory.to_path_buf()]; + while let Some(root) = pending.pop() { + let Ok(entries) = std::fs::read_dir(&root) else { + continue; + }; + let mut files: Vec = Vec::new(); + let mut directories: Vec = Vec::new(); + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + match std::fs::metadata(&path) { + Ok(metadata) if metadata.is_dir() => { + if !entry.file_type().is_ok_and(|kind| kind.is_symlink()) { + directories.push(path); + } + } + _ => files.push(path), + } + } + files.sort(); + directories.sort(); + for file in files { + let relative: String = file + .strip_prefix(directory) + .map_or_else(|_| file.clone(), Path::to_path_buf) + .to_string_lossy() + .into_owned(); + contents.push((relative, contents_digest(&file, rules))); + } + pending.extend(directories.into_iter().rev()); + } + contents +} + +/// Stat every listed module file the project walk does not cover, sorted by path. +pub(crate) fn module_stamps( + module_paths: &[String], + covered: &HashSet, +) -> Vec<(String, i64, u64)> { + let mut seen: HashSet<&str> = HashSet::new(); + let mut stamps: Vec<(String, i64, u64)> = Vec::new(); + for path in module_paths { + if covered.contains(path) || !seen.insert(path.as_str()) { + continue; + } + if let Ok(metadata) = std::fs::metadata(text_path(path)) { + stamps.push((path.clone(), modified_ns(&metadata), metadata.len())); + } + } + stamps.sort(); + stamps +} + +/// Whether every recorded module file still has the same stat identity. +pub(crate) fn module_stamps_unchanged(stamps: &[(String, i64, u64)]) -> bool { + stamps.iter().all( + |(path, mtime_ns, size)| match std::fs::metadata(text_path(path)) { + Ok(metadata) => modified_ns(&metadata) == *mtime_ns && metadata.len() == *size, + Err(_) => false, + }, + ) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs new file mode 100644 index 0000000000..a68068cda7 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod attempt; +pub(crate) mod entry; +pub(crate) mod files; +pub(crate) mod identity; +pub(crate) mod target; +pub(crate) mod timings; diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs new file mode 100644 index 0000000000..ef404d8c4e --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs @@ -0,0 +1,146 @@ +//! Stat identity of the artifacts a compile leaves in target/. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use crate::project_reuse::_helpers::files::file_digest; +use crate::project_reuse::models::{CompileRecord, RecordedArtifact, ReuseRules}; +use crate::project_snapshot::main::stamp_path::stamp_path; +use crate::project_snapshot::models::PathStamp; + +type Stamps = BTreeMap; + +fn path_stamp(path: &str) -> Option { + let mut stamp: PathStamp = stamp_path(Path::new(path))?; + path.clone_into(&mut stamp.relative_path); + Some(stamp) +} + +fn compiled_root(project_dir: &Path, rules: &ReuseRules) -> PathBuf { + project_dir.join(&rules.compiled_root) +} + +/// Stat every compiled artifact plus `extra_paths`, keyed by path. +fn snapshot_target_files(project_dir: &Path, extra_paths: &[&str], rules: &ReuseRules) -> Stamps { + let mut stamps: Stamps = BTreeMap::new(); + let mut pending: Vec = vec![compiled_root(project_dir, rules)]; + while let Some(directory) = pending.pop() { + let Ok(entries) = std::fs::read_dir(&directory) else { + continue; + }; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + if entry.file_type().is_ok_and(|kind| kind.is_dir()) { + pending.push(path); + continue; + } + let path: String = path.to_string_lossy().into_owned(); + if let Some(stamp) = path_stamp(&path) { + let _ = stamps.insert(path, stamp); + } + } + } + for path in extra_paths { + if let Some(stamp) = path_stamp(path) { + let _ = stamps.insert((*path).to_owned(), stamp); + } + } + stamps +} + +fn keyed_stamp(path: &str) -> Option<(String, PathStamp)> { + let stamp: PathStamp = path_stamp(path)?; + Some((path.to_owned(), stamp)) +} + +/// The whole compiled tree plus extra paths, or only the given paths. +fn target_stamps(project_dir: &Path, paths: &[&str], tree: bool, rules: &ReuseRules) -> Stamps { + if !tree { + return paths.iter().copied().filter_map(keyed_stamp).collect(); + } + let prefix: String = compiled_prefix(project_dir, rules); + let extra: Vec<&str> = paths + .iter() + .copied() + .filter(|path| !path.starts_with(prefix.as_str())) + .collect(); + snapshot_target_files(project_dir, &extra, rules) +} + +fn compiled_prefix(project_dir: &Path, rules: &ReuseRules) -> String { + format!( + "{}{}", + compiled_root(project_dir, rules).to_string_lossy(), + std::path::MAIN_SEPARATOR + ) +} + +fn matches_recorded( + stamp: &PathStamp, + artifact: &RecordedArtifact, + since_ns: i64, + rules: &ReuseRules, +) -> bool { + match artifact { + RecordedArtifact::Kept { size, mtime_ns } => { + stamp.size == *size && stamp.mtime_ns == *mtime_ns + } + RecordedArtifact::Written { digest } => { + stamp.mtime_ns.max(stamp.ctime_ns) < since_ns - rules.racy_window_ns + || file_digest(Path::new(&stamp.relative_path)).as_deref() == Some(digest) + } + } +} + +/// Stamp this compile's artifacts, or `None` when any changed since it wrote them. +pub(crate) fn verified_target_files( + project_dir: &Path, + record: &CompileRecord, + since_ns: i64, + rules: &ReuseRules, +) -> Option> { + let prefix: String = compiled_prefix(project_dir, rules); + let dag_path: Option<&str> = record.dag_path.as_deref(); + let written: bool = record.artifacts_written; + let expected: BTreeMap<&str, &RecordedArtifact> = record + .artifacts + .iter() + .filter(|(path, _)| path.starts_with(prefix.as_str()) || Some(path.as_str()) == dag_path) + .map(|(path, artifact)| (path.as_str(), artifact)) + .collect(); + let paths: Vec<&str> = expected.keys().copied().collect(); + let stamps: Stamps = target_stamps(project_dir, &paths, written, rules); + let keys_match: bool = stamps.len() == expected.len() + && stamps + .keys() + .all(|path| expected.contains_key(path.as_str())); + if !keys_match + || !stamps + .iter() + .all(|(path, stamp)| matches_recorded(stamp, expected[path.as_str()], since_ns, rules)) + { + return None; + } + if target_stamps(project_dir, &paths, written, rules) != stamps { + return None; + } + Some(stamps.into_values().collect()) +} + +/// Whether every stored artifact is unchanged and, for a written tree, none was added. +pub(crate) fn target_files_unchanged( + stored: &[PathStamp], + project_dir: &Path, + tree: bool, + rules: &ReuseRules, +) -> bool { + let paths: Vec<&str> = stored + .iter() + .map(|stamp| stamp.relative_path.as_str()) + .collect(); + let expected: Stamps = stored + .iter() + .map(|stamp| (stamp.relative_path.clone(), stamp.clone())) + .collect(); + target_stamps(project_dir, &paths, tree, rules) == expected +} diff --git a/crates/sqlbuild-cache/src/project_reuse/_helpers/timings.rs b/crates/sqlbuild-cache/src/project_reuse/_helpers/timings.rs new file mode 100644 index 0000000000..51b8edb63c --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/_helpers/timings.rs @@ -0,0 +1,55 @@ +//! Locate and replace the compile_timings object inside a stored JSON report. + +const OPENING: &str = "\n \"compile_timings\": {"; +const CLOSING: &str = "\n }"; + +/// Byte span of the top-level compile_timings object, if any. +pub(crate) fn compile_timings_span(stdout: &str) -> Option<(usize, usize)> { + let opening: usize = stdout.find(OPENING)?; + let start: usize = opening + OPENING.len() - 1; + let closing: usize = start + stdout[start..].find(CLOSING)?; + if stdout[start + 1..closing].contains('{') { + return None; + } + Some((start, closing + CLOSING.len())) +} + +/// Splice freshly measured timings into a stored two-space-indented JSON report. +pub(crate) fn replace_compile_timings( + stdout: &str, + span: (usize, usize), + timings: &[(String, i64)], +) -> Option { + let rendered: Vec = timings + .iter() + .map(|(name, value)| format!(" {}: {value}", json_string(name))) + .collect(); + let head: &str = stdout.get(..span.0)?; + let tail: &str = stdout.get(span.1..)?; + Some(format!("{head}{{\n{}{CLOSING}{tail}", rendered.join(",\n"))) +} + +/// `json.dumps(name)` for a string: ASCII-escaped and double-quoted. +fn json_string(text: &str) -> String { + let mut quoted: String = String::from("\""); + for character in text.chars() { + match character { + '"' => quoted.push_str("\\\""), + '\\' => quoted.push_str("\\\\"), + '\n' => quoted.push_str("\\n"), + '\r' => quoted.push_str("\\r"), + '\t' => quoted.push_str("\\t"), + '\u{08}' => quoted.push_str("\\b"), + '\u{0c}' => quoted.push_str("\\f"), + ' '..='~' => quoted.push(character), + _ => { + let mut units: [u16; 2] = [0; 2]; + for unit in character.encode_utf16(&mut units) { + quoted.push_str(&format!("\\u{unit:04x}")); + } + } + } + } + quoted.push('"'); + quoted +} diff --git a/crates/sqlbuild-cache/src/project_reuse/constants.rs b/crates/sqlbuild-cache/src/project_reuse/constants.rs new file mode 100644 index 0000000000..1c61e3745a --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/constants.rs @@ -0,0 +1,8 @@ +/// The native store kind holding one stored compile per selected target. +pub const REUSE_STORE_KIND: &str = "project-reuse"; +/// The store environment; bumped when the stored records change shape. +pub const REUSE_STORE_ENVIRONMENT: &str = "project-reuse-records-1"; +/// Suffix of every reuse store file. +pub const REUSE_STORE_SUFFIX: &str = "store"; +/// Prefix of digests computed natively, so they never equal a Python digest. +pub const NATIVE_DIGEST_PREFIX: &str = "b3:"; diff --git a/crates/sqlbuild-cache/src/project_reuse/errors.rs b/crates/sqlbuild-cache/src/project_reuse/errors.rs new file mode 100644 index 0000000000..8d547eb166 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/errors.rs @@ -0,0 +1,15 @@ +//! Why a stored compile could not be recorded. + +/// Recording failed; the stored compile for this slot was removed. +#[derive(Debug)] +pub struct ReuseRecordError { + pub message: String, +} + +impl From for ReuseRecordError { + fn from(error: std::io::Error) -> Self { + Self { + message: error.to_string(), + } + } +} diff --git a/crates/sqlbuild-cache/src/project_reuse/main/check_reuse.rs b/crates/sqlbuild-cache/src/project_reuse/main/check_reuse.rs new file mode 100644 index 0000000000..bf2412be49 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/main/check_reuse.rs @@ -0,0 +1,6 @@ +use crate::project_reuse::models::{ReuseAttempt, ReuseCheck}; + +/// Replay the stored compile when every input matches, otherwise prepare to store one. +pub fn check_reuse(check: ReuseCheck<'_>) -> ReuseAttempt { + crate::project_reuse::_helpers::attempt::check(check) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/main/mod.rs b/crates/sqlbuild-cache/src/project_reuse/main/mod.rs new file mode 100644 index 0000000000..f6d51418fe --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/main/mod.rs @@ -0,0 +1,4 @@ +pub mod check_reuse; +pub mod pending_digest_bytes; +pub mod record_reuse; +pub mod replay_stdout; diff --git a/crates/sqlbuild-cache/src/project_reuse/main/pending_digest_bytes.rs b/crates/sqlbuild-cache/src/project_reuse/main/pending_digest_bytes.rs new file mode 100644 index 0000000000..f45a1beccf --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/main/pending_digest_bytes.rs @@ -0,0 +1,10 @@ +use crate::project_reuse::models::ReuseAttempt; + +/// Bytes recording this attempt would hash for files whose stamp moved. +pub fn pending_digest_bytes(attempt: &ReuseAttempt) -> u64 { + crate::project_reuse::_helpers::files::pending_digest_bytes( + &attempt.snapshot, + &attempt.digests, + &attempt.restamped, + ) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/main/record_reuse.rs b/crates/sqlbuild-cache/src/project_reuse/main/record_reuse.rs new file mode 100644 index 0000000000..5237946cf2 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/main/record_reuse.rs @@ -0,0 +1,11 @@ +use crate::project_reuse::errors::ReuseRecordError; +use crate::project_reuse::models::{CompileRecord, RecordResult, ReuseAttempt, ReuseRules}; + +/// Store a reusable full compile, or clear the slot when this compile cannot be reused. +pub fn record_reuse( + attempt: &ReuseAttempt, + record: &CompileRecord, + rules: &ReuseRules, +) -> Result { + crate::project_reuse::_helpers::attempt::record(attempt, record, rules) +} diff --git a/crates/sqlbuild-cache/src/project_reuse/main/replay_stdout.rs b/crates/sqlbuild-cache/src/project_reuse/main/replay_stdout.rs new file mode 100644 index 0000000000..4c5ca5c1dd --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/main/replay_stdout.rs @@ -0,0 +1,13 @@ +use crate::project_reuse::models::StoredOutput; + +/// The stored report with freshly measured timings, or `None` when it cannot be spliced. +pub fn replay_stdout(output: &StoredOutput, timings: &[(String, i64)]) -> Option { + match output.timings_span { + None => Some(output.stdout.clone()), + Some(span) => crate::project_reuse::_helpers::timings::replace_compile_timings( + &output.stdout, + span, + timings, + ), + } +} diff --git a/crates/sqlbuild-cache/src/project_reuse/mod.rs b/crates/sqlbuild-cache/src/project_reuse/mod.rs new file mode 100644 index 0000000000..2a66cc14e3 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/mod.rs @@ -0,0 +1,10 @@ +//! Whole-project compile reuse: the stored compile, its input identity and its replay. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/project_reuse/models.rs b/crates/sqlbuild-cache/src/project_reuse/models.rs new file mode 100644 index 0000000000..7a2ffde470 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/models.rs @@ -0,0 +1,166 @@ +//! The stored compile, the facts it is checked against, and one invocation's reuse state. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::PathBuf; + +use crate::project_snapshot::models::{PathStamp, SnapshotRules}; + +/// Constants the Python host owns, passed once so both sides agree. +#[derive(Clone, Debug, Default)] +pub struct ReuseRules { + pub snapshot: SnapshotRules, + pub tracked_prefixes: Vec, + pub untracked_names: Vec, + /// Environment variables that turn reuse off when set to their value. + pub disabling_variables: Vec<(String, String)>, + pub config_filenames: Vec, + pub racy_window_ns: i64, + pub max_stored_entries: usize, + pub max_entry_bytes: usize, + pub missing_environment_value: String, + pub missing_file_digest: String, + pub missing_path_mtime_ns: i64, + pub project_root_path_mtime_ns: i64, + pub compiled_root: PathBuf, + /// Directories beside the store directory that older releases kept stored compiles in. + pub retired_directories: Vec, +} + +/// Interpreter and invocation facts only the Python host can read. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct HostIdentity { + pub invocation_digest: String, + pub runtime: BTreeMap, + /// `sys.path` in order; stamped natively. + pub search_path: Vec, +} + +/// Where one provider settings class reads values outside the project files. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SettingsInputs { + pub case_sensitive: bool, + pub names: Vec, + pub prefixes: Vec, + pub env_files: Vec, + pub secrets_dirs: Vec, +} + +/// One project path recorded before the stored compile ran. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredProjectFile { + pub stamp: PathStamp, + pub digest: Option, + pub racy: bool, +} + +/// Every input identity recorded with one stored compile. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StoredInputs { + pub invocation_digest: String, + pub runtime: BTreeMap, + pub search_path: Vec<(String, i64)>, + pub environment_names: Vec, + pub environment_digest: String, + pub modules: Vec<(String, i64, u64)>, + pub project_files: Vec, + pub target_files: Vec, + pub target_tree: bool, + pub settings_inputs: Vec, + pub settings_digest: String, +} + +/// The command output of one stored compile. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct StoredOutput { + pub stderr_lines: Vec, + pub exit_code: i64, + /// Byte span of the `compile_timings` object in `stdout`. + pub timings_span: Option<(usize, usize)>, + pub stdout: String, +} + +/// Whether one invocation replays, compiles and stores, or bypasses reuse. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReuseOutcome { + Hit, + Miss, + Bypass, +} + +/// One invocation's reuse state, carried from the check to the record. +#[derive(Debug)] +pub struct ReuseAttempt { + pub outcome: ReuseOutcome, + pub project_dir: PathBuf, + pub store_path: PathBuf, + pub identity: HostIdentity, + pub search_path: Vec<(String, i64)>, + pub snapshot: Vec, + pub snapshot_ns: i64, + pub digests: HashMap, + pub restamped: HashSet, + /// The stored output to replay on a hit. + pub replay: Option, + /// A stored compile matched but its output could not be replayed. + pub replay_failed: bool, + /// Project files the check read to compare or complete content digests. + pub digested_files: usize, +} + +/// What the host asks of one reuse check. +#[derive(Debug)] +pub struct ReuseCheck<'a> { + pub project_dir: PathBuf, + pub store_path: PathBuf, + pub identity: HostIdentity, + /// The command line turns reuse off (no cache, manifest, profiling or debug). + pub bypass_requested: bool, + pub json_output: bool, + pub rules: &'a ReuseRules, +} + +/// What a full compile leaves for the record step. +#[derive(Debug)] +pub struct CompileRecord { + pub stdout: Option, + pub stderr_lines: Vec, + pub exit_code: i64, + pub compile_cache_enabled: bool, + pub read_run_id: bool, + /// Provider settings inputs, or why they cannot be enumerated. + pub settings: Result, String>, + pub template_environment_names: Vec, + /// Loaded module files and extra module paths, as the host lists them. + pub module_paths: Vec, + /// Artifacts the compile wrote (digest) or kept (size and mtime), by absolute path. + pub artifacts: HashMap, + pub artifacts_written: bool, + pub dag_path: Option, + pub json_output: bool, +} + +/// What one compile left at one artifact path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RecordedArtifact { + Written { digest: String }, + Kept { size: u64, mtime_ns: i64 }, +} + +/// What recording did, and how many project files it read for content digests. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RecordResult { + pub outcome: RecordOutcome, + pub digested_files: usize, +} + +/// Whether recording stored the compile. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecordOutcome { + /// Nothing was recorded for this invocation. + NotAttempted, + Stored, + /// The slot was cleared because this compile cannot be reused. + Removed, + /// Files changed while the compile ran; the slot was cleared. + Skipped, +} diff --git a/crates/sqlbuild-cache/src/project_reuse/tests.rs b/crates/sqlbuild-cache/src/project_reuse/tests.rs new file mode 100644 index 0000000000..98085a469c --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_file_changes.rs"] +mod file_changes; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_project_reuse.rs"] +mod project_reuse; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs b/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs new file mode 100644 index 0000000000..3bc8626078 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/tests/helpers.rs @@ -0,0 +1,98 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use crate::project_reuse::_helpers::entry::write_entry; +use crate::project_reuse::models::{SettingsInputs, StoredInputs, StoredOutput, StoredProjectFile}; +use crate::project_snapshot::models::PathStamp; + +pub(crate) fn stamp(relative_path: &str, kind: &'static str, link: Option<&str>) -> PathStamp { + PathStamp { + relative_path: relative_path.to_owned(), + kind, + size: 42, + mtime_ns: -7, + ctime_ns: 1_700_000_000_000_000_000, + inode: 9, + link: link.map(str::to_owned), + } +} + +pub(crate) fn stored_inputs() -> StoredInputs { + StoredInputs { + invocation_digest: "b3:orders".to_owned(), + runtime: BTreeMap::from([("python".to_owned(), "3.12".to_owned())]), + search_path: vec![("site".to_owned(), -2)], + environment_names: vec!["SQLBUILD_REGION".to_owned()], + environment_digest: "b3:env".to_owned(), + modules: vec![("/site/orders.py".to_owned(), 5, 10)], + project_files: vec![ + StoredProjectFile { + stamp: stamp("models/orders.sql", "f", None), + digest: Some("b3:model".to_owned()), + racy: true, + }, + StoredProjectFile { + stamp: stamp("seeds", "dl", Some("../shared")), + digest: None, + racy: false, + }, + ], + target_files: vec![stamp("/project/target/compiled/orders.sql", "f", None)], + target_tree: true, + settings_inputs: vec![SettingsInputs { + case_sensitive: true, + names: vec!["region".to_owned()], + prefixes: vec!["orders_".to_owned()], + env_files: vec![".env".to_owned()], + secrets_dirs: vec!["/secrets".to_owned()], + }], + settings_digest: "b3:settings".to_owned(), + } +} + +pub(crate) fn stored_output() -> StoredOutput { + StoredOutput { + stderr_lines: vec!["Compiled 1 model".to_owned()], + exit_code: 1, + timings_span: Some((3, 9)), + stdout: "{\"command\": \"compile\"}\n".to_owned(), + } +} + +/// Store `slots` compiles in `directory`, each older than the next, keeping `max_entries`. +pub(crate) fn store_aged_slots(directory: &Path, slots: u64, max_entries: usize) { + for slot in 0..slots { + let path = directory.join(format!("slot{slot}.store")); + write_entry(&path, &stored_inputs(), &stored_output(), max_entries).expect("stored"); + let stamp = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(slot + 1); + std::fs::File::options() + .write(true) + .open(&path) + .and_then(|file| file.set_modified(stamp)) + .expect("aged slot"); + } +} + +/// The slot file names left in `directory`, sorted. +pub(crate) fn slot_names(directory: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(directory) + .expect("slots") + .map(|entry| { + entry + .expect("slot") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + names.sort(); + names +} + +/// Flip one byte near the end of the file. +pub(crate) fn damage(path: &Path) { + let mut bytes = std::fs::read(path).expect("slot bytes"); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + std::fs::write(path, bytes).expect("damaged slot"); +} diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_file_changes.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_file_changes.rs new file mode 100644 index 0000000000..09358c1ac4 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_file_changes.rs @@ -0,0 +1,171 @@ +use crate::project_reuse::_helpers::files::{compare_project_files, file_digest}; +use crate::project_reuse::models::StoredProjectFile; +use crate::project_reuse::tests::test_types::FileComparisonTestCase; +#[cfg(unix)] +use crate::project_reuse::tests::test_types::{LinkChangeTestCase, LinkCycleTestCase}; +use crate::project_snapshot::main::snapshot_project_files::snapshot_project_files; +use crate::project_snapshot::models::SnapshotRules; + +#[test] +fn given_recorded_file_when_stat_is_equal_or_moves_then_racy_content_is_verified() { + let test_cases = [ + FileComparisonTestCase { + description: "racy equal stamps hide changed bytes", + racy: true, + has_digest: true, + moved_stamp: false, + contents: b"SELECT 2", + expected_unchanged: false, + expected_reads: 1, + }, + FileComparisonTestCase { + description: "racy equal stamps preserve unchanged bytes", + racy: true, + has_digest: true, + moved_stamp: false, + contents: b"SELECT 1", + expected_unchanged: true, + expected_reads: 1, + }, + FileComparisonTestCase { + description: "settled equal stamps are trusted", + racy: false, + has_digest: true, + moved_stamp: false, + contents: b"SELECT 2", + expected_unchanged: true, + expected_reads: 0, + }, + FileComparisonTestCase { + description: "changed stat-only record misses", + racy: false, + has_digest: false, + moved_stamp: true, + contents: b"SELECT 1", + expected_unchanged: false, + expected_reads: 0, + }, + FileComparisonTestCase { + description: "changed stamps with equal bytes reuse", + racy: false, + has_digest: true, + moved_stamp: true, + contents: b"SELECT 1", + expected_unchanged: true, + expected_reads: 1, + }, + ]; + for test_case in test_cases { + let folder = tempfile::tempdir().expect("project"); + let path = folder.path().join("orders.sql"); + std::fs::write(&path, b"SELECT 1").expect("original model"); + let mut current = snapshot_project_files(folder.path(), &SnapshotRules::default()); + let stored = vec![StoredProjectFile { + stamp: current[0].clone(), + digest: test_case + .has_digest + .then(|| file_digest(&path).expect("digest")), + racy: test_case.racy, + }]; + std::fs::write(&path, test_case.contents).expect("current bytes"); + current[0].mtime_ns += i64::from(test_case.moved_stamp); + let (unchanged, _, reads) = compare_project_files(folder.path(), &stored, ¤t); + assert_eq!( + unchanged, test_case.expected_unchanged, + "{}", + test_case.description + ); + assert_eq!(reads, test_case.expected_reads, "{}", test_case.description); + } +} + +#[cfg(unix)] +#[test] +fn given_linked_inputs_when_targets_change_then_compile_reuse_is_invalidated() { + use std::os::unix::fs::symlink; + + let test_cases = [ + LinkChangeTestCase { + description: "edit file target", + change: |_, outside| { + std::fs::write(outside.join("orders.sql"), b"SELECT 2").expect("edited target"); + }, + expected_unchanged: false, + }, + LinkChangeTestCase { + description: "retarget file link", + change: |project, outside| { + std::fs::remove_file(project.join("orders.sql")).expect("unlink"); + symlink(outside.join("other.sql"), project.join("orders.sql")).expect("retarget"); + }, + expected_unchanged: false, + }, + LinkChangeTestCase { + description: "add file through directory link", + change: |_, outside| { + std::fs::write(outside.join("models/customers.sql"), b"SELECT 1") + .expect("added model"); + }, + expected_unchanged: false, + }, + LinkChangeTestCase { + description: "break file link", + change: |_, outside| { + std::fs::remove_file(outside.join("orders.sql")).expect("removed target"); + }, + expected_unchanged: false, + }, + ]; + for test_case in test_cases { + let project = tempfile::tempdir().expect("project"); + let outside = tempfile::tempdir().expect("linked resources"); + std::fs::create_dir(outside.path().join("models")).expect("model directory"); + std::fs::write(outside.path().join("orders.sql"), b"SELECT 1").expect("target"); + std::fs::write(outside.path().join("other.sql"), b"SELECT 1").expect("other target"); + symlink( + outside.path().join("orders.sql"), + project.path().join("orders.sql"), + ) + .expect("file link"); + symlink(outside.path().join("models"), project.path().join("models")) + .expect("directory link"); + let stored: Vec = + snapshot_project_files(project.path(), &SnapshotRules::default()) + .into_iter() + .map(|stamp| StoredProjectFile { + digest: file_digest(&project.path().join(&stamp.relative_path)), + stamp, + racy: true, + }) + .collect(); + (test_case.change)(project.path(), outside.path()); + let current = snapshot_project_files(project.path(), &SnapshotRules::default()); + assert_eq!( + compare_project_files(project.path(), &stored, ¤t).0, + test_case.expected_unchanged, + "{}", + test_case.description + ); + } +} + +#[cfg(unix)] +#[test] +fn given_directory_link_to_project_root_when_snapshotting_then_cycle_is_recorded_once() { + let test_cases = [LinkCycleTestCase { + description: "directory link back to project root", + expected_paths: &["loop", "orders.sql"], + }]; + for test_case in test_cases { + let project = tempfile::tempdir().expect("project"); + std::fs::write(project.path().join("orders.sql"), b"SELECT 1").expect("model"); + std::os::unix::fs::symlink(project.path(), project.path().join("loop")).expect("cycle"); + let snapshot = snapshot_project_files(project.path(), &SnapshotRules::default()); + let mut paths: Vec<&str> = snapshot + .iter() + .map(|stamp| stamp.relative_path.as_str()) + .collect(); + paths.sort_unstable(); + assert_eq!(paths, test_case.expected_paths, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs new file mode 100644 index 0000000000..13296a9d8b --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_project_reuse.rs @@ -0,0 +1,215 @@ +use crate::project_reuse::_helpers::entry::{read_entry, write_entry}; +use crate::project_reuse::_helpers::identity::search_path_stamps; +use crate::project_reuse::_helpers::timings::{compile_timings_span, replace_compile_timings}; +use crate::project_reuse::models::ReuseRules; +use crate::project_reuse::tests::helpers::{ + damage, slot_names, store_aged_slots, stored_inputs, stored_output, +}; +use crate::project_reuse::tests::test_types::{ + EntryReadTestCase, NonUnicodeNameTestCase, SlotPruneTestCase, TimingsSpanTestCase, +}; +use crate::project_snapshot::main::path_text::path_text; +use crate::project_snapshot::main::snapshot_project_files::snapshot_project_files; +use crate::project_snapshot::main::text_path::text_path; +use crate::project_snapshot::models::SnapshotRules; + +#[test] +fn given_stored_compile_when_reading_then_only_an_intact_slot_round_trips() { + let test_cases = [ + EntryReadTestCase { + description: "intact slot", + edit: |_| {}, + expected_read: true, + }, + EntryReadTestCase { + description: "damaged slot", + edit: damage, + expected_read: false, + }, + EntryReadTestCase { + description: "truncated slot", + edit: |path| std::fs::write(path, b"SQBSTORE").expect("truncated"), + expected_read: false, + }, + EntryReadTestCase { + description: "missing slot", + edit: |path| std::fs::remove_file(path).expect("removed"), + expected_read: false, + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let path = folder.path().join("reuse/slot.store"); + write_entry(&path, &stored_inputs(), &stored_output(), 8).expect("stored compile"); + (test_case.edit)(&path); + + let read = read_entry(&path); + + assert_eq!( + read, + test_case + .expected_read + .then(|| (stored_inputs(), stored_output())), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_more_slots_than_retained_when_storing_then_the_oldest_are_pruned() { + let test_cases = [ + SlotPruneTestCase { + description: "within the limit", + slots: 2, + max_entries: 8, + expected_kept: &["slot0.store", "slot1.store"], + }, + SlotPruneTestCase { + description: "over the limit", + slots: 4, + max_entries: 2, + expected_kept: &["slot2.store", "slot3.store"], + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("store folder"); + let directory = folder.path().join("reuse"); + + store_aged_slots(&directory, test_case.slots, test_case.max_entries); + + assert_eq!( + slot_names(&directory), + test_case.expected_kept, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_report_when_replaying_then_only_a_flat_top_level_timings_object_is_replaced() { + let test_cases = [ + TimingsSpanTestCase { + description: "flat timings object", + stdout: "{\n \"compile_timings\": {\n \"total_ms\": 900\n },\n \"models\": []\n}\n", + expected_replayed: Some( + "{\n \"compile_timings\": {\n \"project_reuse_hits\": 1,\n \"total_ms\": 4\n },\n \"models\": []\n}\n", + ), + }, + TimingsSpanTestCase { + description: "nested timings object", + stdout: "{\n \"compile_timings\": {\n \"phases\": {}\n }\n}\n", + expected_replayed: None, + }, + TimingsSpanTestCase { + description: "no timings object", + stdout: "{\n \"models\": []\n}\n", + expected_replayed: None, + }, + ]; + + for test_case in test_cases { + let timings = [ + ("project_reuse_hits".to_owned(), 1), + ("total_ms".to_owned(), 4), + ]; + + let replayed = compile_timings_span(test_case.stdout) + .and_then(|span| replace_compile_timings(test_case.stdout, span, &timings)); + + assert_eq!( + replayed.as_deref(), + test_case.expected_replayed, + "{}", + test_case.description + ); + } +} + +#[cfg(unix)] +#[test] +fn given_file_names_when_snapshotting_then_each_stamp_names_its_exact_file() { + use std::os::unix::ffi::OsStrExt; + + let test_cases = [ + NonUnicodeNameTestCase { + description: "unicode name", + name: "orders_é.sql".as_bytes(), + expected_escaped: false, + }, + NonUnicodeNameTestCase { + description: "name that is not UTF-8", + name: b"orders_\xff.sql", + expected_escaped: true, + }, + ]; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("project folder"); + let name = std::ffi::OsStr::from_bytes(test_case.name); + std::fs::write(folder.path().join(name), b"SELECT 1").expect("model file"); + + let stamps = snapshot_project_files(folder.path(), &SnapshotRules::default()); + + assert_eq!(stamps.len(), 1, "{}", test_case.description); + assert_eq!( + stamps[0].relative_path.starts_with('\0'), + test_case.expected_escaped, + "{}", + test_case.description + ); + assert_eq!( + std::fs::read(folder.path().join(text_path(&stamps[0].relative_path))) + .expect("file named by the stamp"), + b"SELECT 1", + "{}", + test_case.description + ); + } +} + +#[cfg(unix)] +#[test] +fn given_search_path_entry_that_is_not_utf8_when_stamping_then_its_real_folder_is_stamped() { + use std::os::unix::ffi::OsStrExt; + + let test_cases = [ + NonUnicodeNameTestCase { + description: "unicode folder", + name: "site_é".as_bytes(), + expected_escaped: false, + }, + NonUnicodeNameTestCase { + description: "folder that is not UTF-8", + name: b"site_\xff", + expected_escaped: true, + }, + ]; + let rules = ReuseRules { + missing_path_mtime_ns: -1, + project_root_path_mtime_ns: -2, + ..ReuseRules::default() + }; + + for test_case in test_cases { + let folder = tempfile::tempdir().expect("search path parent"); + let entry = folder + .path() + .join(std::ffi::OsStr::from_bytes(test_case.name)); + std::fs::create_dir(&entry).expect("search path folder"); + let text = path_text(entry.as_os_str()); + + let stamps = search_path_stamps(std::slice::from_ref(&text), folder.path(), &rules); + + assert_eq!( + text.starts_with('\0'), + test_case.expected_escaped, + "{}", + test_case.description + ); + assert!(stamps[0].1 > 0, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs b/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs new file mode 100644 index 0000000000..b1369a1620 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_reuse/tests/test_types.rs @@ -0,0 +1,52 @@ +use std::path::Path; + +/// One stored slot edited before it is read back. +pub(crate) struct EntryReadTestCase { + pub(crate) description: &'static str, + pub(crate) edit: fn(&Path), + pub(crate) expected_read: bool, +} + +/// Slots stored oldest first under one retention limit. +pub(crate) struct SlotPruneTestCase { + pub(crate) description: &'static str, + pub(crate) slots: u64, + pub(crate) max_entries: usize, + pub(crate) expected_kept: &'static [&'static str], +} + +/// One report and the compile_timings span expected inside it. +pub(crate) struct TimingsSpanTestCase { + pub(crate) description: &'static str, + pub(crate) stdout: &'static str, + pub(crate) expected_replayed: Option<&'static str>, +} + +/// A project file name and whether the walk keeps its exact bytes. +pub(crate) struct NonUnicodeNameTestCase { + pub(crate) description: &'static str, + pub(crate) name: &'static [u8], + pub(crate) expected_escaped: bool, +} +pub(crate) struct FileComparisonTestCase { + pub(crate) description: &'static str, + pub(crate) racy: bool, + pub(crate) has_digest: bool, + pub(crate) moved_stamp: bool, + pub(crate) contents: &'static [u8], + pub(crate) expected_unchanged: bool, + pub(crate) expected_reads: usize, +} + +#[cfg(unix)] +pub(crate) struct LinkChangeTestCase { + pub(crate) description: &'static str, + pub(crate) change: fn(&std::path::Path, &std::path::Path), + pub(crate) expected_unchanged: bool, +} + +#[cfg(unix)] +pub(crate) struct LinkCycleTestCase { + pub(crate) description: &'static str, + pub(crate) expected_paths: &'static [&'static str], +} diff --git a/crates/sqlbuild-cache/src/project_snapshot/_helpers/mod.rs b/crates/sqlbuild-cache/src/project_snapshot/_helpers/mod.rs new file mode 100644 index 0000000000..0cf65edbc4 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod path_text; +pub(crate) mod stat; +pub(crate) mod walk; diff --git a/crates/sqlbuild-cache/src/project_snapshot/_helpers/path_text.rs b/crates/sqlbuild-cache/src/project_snapshot/_helpers/path_text.rs new file mode 100644 index 0000000000..e80a635e62 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/_helpers/path_text.rs @@ -0,0 +1,100 @@ +//! A path as exact text: UTF-8 as is, anything else escaped so it still names the same file. + +use std::ffi::{OsStr, OsString}; +use std::path::PathBuf; + +/// Starts every escaped path; no file name contains NUL. +const ESCAPED: char = '\0'; + +/// The path's text, escaping a name that is not valid Unicode. +pub(crate) fn path_text(path: &OsStr) -> String { + match path.to_str() { + Some(text) => text.to_owned(), + None => escaped(path), + } +} + +/// The path `path_text` produced `text` for. +pub(crate) fn text_path(text: &str) -> PathBuf { + match text.strip_prefix(ESCAPED) { + Some(hex) => PathBuf::from(unescaped(hex).unwrap_or_else(|| OsString::from(text))), + None => PathBuf::from(text), + } +} + +fn hex_units(units: impl Iterator, width: usize) -> String { + units.map(|unit| format!("{unit:0width$x}")).collect() +} + +fn parse_units(hex: &str, width: usize) -> Option> { + if !hex.len().is_multiple_of(width) { + return None; + } + (0..hex.len()) + .step_by(width) + .map(|start| parse_unit(hex.get(start..start + width)?)) + .collect() +} + +fn narrow>(unit: u32) -> Option { + let Ok(narrowed) = T::try_from(unit) else { + return None; + }; + Some(narrowed) +} + +fn parse_unit(digits: &str) -> Option { + let Ok(unit) = u32::from_str_radix(digits, 16) else { + return None; + }; + Some(unit) +} + +#[cfg(unix)] +fn escaped(path: &OsStr) -> String { + use std::os::unix::ffi::OsStrExt; + format!( + "{ESCAPED}{}", + hex_units(path.as_bytes().iter().map(|&byte| u32::from(byte)), 2) + ) +} + +#[cfg(unix)] +fn unescaped(hex: &str) -> Option { + use std::os::unix::ffi::OsStringExt; + let bytes: Vec = parse_units(hex, 2)? + .into_iter() + .map(narrow::) + .collect::>>()?; + Some(OsString::from_vec(bytes)) +} + +#[cfg(windows)] +fn escaped(path: &OsStr) -> String { + use std::os::windows::ffi::OsStrExt; + format!( + "{ESCAPED}{}", + hex_units(path.encode_wide().map(u32::from), 4) + ) +} + +#[cfg(windows)] +fn unescaped(hex: &str) -> Option { + use std::os::windows::ffi::OsStringExt; + let units: Vec = parse_units(hex, 4)? + .into_iter() + .map(narrow::) + .collect::>>()?; + Some(OsString::from_wide(&units)) +} + +#[cfg(not(any(unix, windows)))] +fn escaped(path: &OsStr) -> String { + path.to_string_lossy().into_owned() +} + +#[cfg(not(any(unix, windows)))] +fn unescaped(hex: &str) -> Option { + let _ = hex; + None +} diff --git a/crates/sqlbuild-cache/src/project_snapshot/_helpers/stat.rs b/crates/sqlbuild-cache/src/project_snapshot/_helpers/stat.rs new file mode 100644 index 0000000000..cb78f7e926 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/_helpers/stat.rs @@ -0,0 +1,70 @@ +//! The stat fields Python's `os.stat_result` gives a project path, per platform. + +use std::fs::Metadata; +use std::path::Path; + +use crate::project_snapshot::models::PathStamp; + +#[cfg(unix)] +const NANOS: i64 = 1_000_000_000; + +/// `(st_mtime_ns, st_ctime_ns)`. +#[cfg(unix)] +pub(crate) fn stat_times(metadata: &Metadata) -> (i64, i64) { + use std::os::unix::fs::MetadataExt; + ( + metadata.mtime() * NANOS + metadata.mtime_nsec(), + metadata.ctime() * NANOS + metadata.ctime_nsec(), + ) +} + +/// `(st_mtime_ns, st_ctime_ns)`; Windows reports the creation time as `st_ctime`. +#[cfg(not(unix))] +pub(crate) fn stat_times(metadata: &Metadata) -> (i64, i64) { + ( + system_time_ns(metadata.modified()), + system_time_ns(metadata.created()), + ) +} + +#[cfg(not(unix))] +fn system_time_ns(time: std::io::Result) -> i64 { + let Ok(time) = time else { + return 0; + }; + match time.duration_since(std::time::UNIX_EPOCH) { + Ok(elapsed) => i64::try_from(elapsed.as_nanos()).unwrap_or(i64::MAX), + Err(_) => 0, + } +} + +/// `(st_dev, st_ino)`, or `None` where the platform does not expose them stably. +#[cfg(unix)] +pub(crate) fn file_identity(metadata: &Metadata) -> Option<(u64, u64)> { + use std::os::unix::fs::MetadataExt; + Some((metadata.dev(), metadata.ino())) +} + +/// `(st_dev, st_ino)`, or `None` where the platform does not expose them stably. +#[cfg(not(unix))] +pub(crate) fn file_identity(metadata: &Metadata) -> Option<(u64, u64)> { + let _ = metadata; + None +} + +/// `os.lstat` identity of one path as a file stamp, or `None` when it is missing. +pub(crate) fn stamp_path(path: &Path) -> Option { + let Ok(metadata) = std::fs::symlink_metadata(path) else { + return None; + }; + let (mtime_ns, ctime_ns) = stat_times(&metadata); + Some(PathStamp { + relative_path: path.to_string_lossy().into_owned(), + kind: "f", + size: metadata.len(), + mtime_ns, + ctime_ns, + inode: file_identity(&metadata).map_or(0, |(_, inode)| inode), + link: None, + }) +} diff --git a/crates/sqlbuild-cache/src/project_snapshot/_helpers/walk.rs b/crates/sqlbuild-cache/src/project_snapshot/_helpers/walk.rs new file mode 100644 index 0000000000..1f6502e286 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/_helpers/walk.rs @@ -0,0 +1,131 @@ +//! Python's `snapshot_project_files` walk over `os.scandir`, `os.readlink` and `os.stat`. + +use std::collections::HashSet; +use std::fs::Metadata; +use std::path::{Path, PathBuf}; + +use crate::project_snapshot::_helpers::path_text::path_text; +use crate::project_snapshot::_helpers::stat::{file_identity, stat_times}; +use crate::project_snapshot::models::{PathStamp, SnapshotRules}; + +const FILE: &str = "f"; +const FILE_LINK: &str = "fl"; +const DIRECTORY: &str = "d"; +const DIRECTORY_LINK: &str = "dl"; +const BROKEN_LINK: &str = "bl"; +const SPECIAL: &str = "s"; +const PRESENCE: &str = "p"; + +/// Every stamped path in walk order; names that are not UTF-8 keep their exact bytes escaped. +pub(crate) fn snapshot(project_dir: &Path, rules: &SnapshotRules) -> Vec { + let mut stamps: Vec = Vec::new(); + let mut visited: HashSet = HashSet::from([real_path(project_dir)]); + let mut pending: Vec<(PathBuf, bool)> = vec![(project_dir.to_path_buf(), true)]; + while let Some((directory, is_root)) = pending.pop() { + let Ok(entries) = std::fs::read_dir(&directory) else { + continue; + }; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + let file_name = entry.file_name(); + let name: String = path_text(&file_name); + let Ok(relative) = path.strip_prefix(project_dir) else { + continue; + }; + let relative_path: String = path_text(relative.as_os_str()); + let Some(stamp) = entry_stamp(&path, &name, is_root, rules) else { + continue; + }; + if stamp.kind == DIRECTORY + || (stamp.kind == DIRECTORY_LINK && visited.insert(real_path(&path))) + { + pending.push((path, false)); + } + stamps.push(PathStamp { + relative_path, + ..stamp + }); + } + } + stamps +} + +fn real_path(path: &Path) -> PathBuf { + std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +/// The entry's stamp, or `None` to skip it. +fn entry_stamp(path: &Path, name: &str, is_root: bool, rules: &SnapshotRules) -> Option { + let Ok(metadata) = std::fs::symlink_metadata(path) else { + return None; + }; + if metadata.file_type().is_symlink() { + return link_stamp(path, name, is_root, rules); + } + if metadata.is_dir() { + return (!excluded(name, is_root, rules)).then(|| marker(DIRECTORY, None)); + } + if rules + .presence_suffixes + .iter() + .any(|suffix| name.ends_with(suffix.as_str())) + { + return Some(marker(PRESENCE, None)); + } + if file_identity(&metadata).is_some_and(|identity| rules.output_files.contains(&identity)) { + return Some(marker(PRESENCE, None)); + } + Some(file_stamp(&metadata, None)) +} + +fn link_stamp(path: &Path, name: &str, is_root: bool, rules: &SnapshotRules) -> Option { + let Ok(target) = std::fs::read_link(path) else { + return None; + }; + let link: String = path_text(target.as_os_str()); + let Ok(metadata) = std::fs::metadata(path) else { + return Some(marker(BROKEN_LINK, Some(link))); + }; + if !metadata.is_dir() { + return Some(file_stamp(&metadata, Some(link))); + } + if excluded(name, is_root, rules) { + return None; + } + Some(marker(DIRECTORY_LINK, Some(link))) +} + +fn excluded(name: &str, is_root: bool, rules: &SnapshotRules) -> bool { + rules.excluded.iter().any(|excluded| excluded == name) + || (is_root && rules.excluded_root.iter().any(|excluded| excluded == name)) +} + +fn marker(kind: &'static str, link: Option) -> PathStamp { + PathStamp { + relative_path: String::new(), + kind, + size: 0, + mtime_ns: 0, + ctime_ns: 0, + inode: 0, + link, + } +} + +fn file_stamp(metadata: &Metadata, link: Option) -> PathStamp { + let kind: &'static str = match (metadata.is_file(), link.is_some()) { + (true, false) => FILE, + (true, true) => FILE_LINK, + (false, _) => SPECIAL, + }; + let (mtime_ns, ctime_ns) = stat_times(metadata); + PathStamp { + relative_path: String::new(), + kind, + size: metadata.len(), + mtime_ns, + ctime_ns, + inode: file_identity(metadata).map_or(0, |(_, inode)| inode), + link, + } +} diff --git a/crates/sqlbuild-cache/src/project_snapshot/main/mod.rs b/crates/sqlbuild-cache/src/project_snapshot/main/mod.rs new file mode 100644 index 0000000000..4db5bd1486 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/main/mod.rs @@ -0,0 +1,4 @@ +pub mod path_text; +pub mod snapshot_project_files; +pub mod stamp_path; +pub mod text_path; diff --git a/crates/sqlbuild-cache/src/project_snapshot/main/path_text.rs b/crates/sqlbuild-cache/src/project_snapshot/main/path_text.rs new file mode 100644 index 0000000000..5405b764d7 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/main/path_text.rs @@ -0,0 +1,6 @@ +use std::ffi::OsStr; + +/// The path's exact text: UTF-8 as is, any other name escaped so it still names the same file. +pub fn path_text(path: &OsStr) -> String { + crate::project_snapshot::_helpers::path_text::path_text(path) +} diff --git a/crates/sqlbuild-cache/src/project_snapshot/main/snapshot_project_files.rs b/crates/sqlbuild-cache/src/project_snapshot/main/snapshot_project_files.rs new file mode 100644 index 0000000000..db59c3bfa2 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/main/snapshot_project_files.rs @@ -0,0 +1,8 @@ +use std::path::Path; + +use crate::project_snapshot::models::{PathStamp, SnapshotRules}; + +/// Stat every compile-relevant path under `project_dir`, in walk order. +pub fn snapshot_project_files(project_dir: &Path, rules: &SnapshotRules) -> Vec { + crate::project_snapshot::_helpers::walk::snapshot(project_dir, rules) +} diff --git a/crates/sqlbuild-cache/src/project_snapshot/main/stamp_path.rs b/crates/sqlbuild-cache/src/project_snapshot/main/stamp_path.rs new file mode 100644 index 0000000000..0d6e10f81b --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/main/stamp_path.rs @@ -0,0 +1,8 @@ +use std::path::Path; + +use crate::project_snapshot::models::PathStamp; + +/// `os.lstat` identity of one artifact path as a file stamp, or `None` when it is missing. +pub fn stamp_path(path: &Path) -> Option { + crate::project_snapshot::_helpers::stat::stamp_path(path) +} diff --git a/crates/sqlbuild-cache/src/project_snapshot/main/text_path.rs b/crates/sqlbuild-cache/src/project_snapshot/main/text_path.rs new file mode 100644 index 0000000000..cc9db8c740 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/main/text_path.rs @@ -0,0 +1,6 @@ +use std::path::PathBuf; + +/// The path `path_text` produced `text` for. +pub fn text_path(text: &str) -> PathBuf { + crate::project_snapshot::_helpers::path_text::text_path(text) +} diff --git a/crates/sqlbuild-cache/src/project_snapshot/mod.rs b/crates/sqlbuild-cache/src/project_snapshot/mod.rs new file mode 100644 index 0000000000..1ad3347f1c --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/mod.rs @@ -0,0 +1,5 @@ +//! The stat snapshot of a project's compile inputs that whole-project reuse compares. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-cache/src/project_snapshot/models.rs b/crates/sqlbuild-cache/src/project_snapshot/models.rs new file mode 100644 index 0000000000..972a553214 --- /dev/null +++ b/crates/sqlbuild-cache/src/project_snapshot/models.rs @@ -0,0 +1,26 @@ +//! Stat identities of project paths, as Python's compile reuse records them. + +/// One path's stat identity: kind code, size, mtime and ctime in nanoseconds, inode and link. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PathStamp { + pub relative_path: String, + pub kind: &'static str, + pub size: u64, + pub mtime_ns: i64, + pub ctime_ns: i64, + pub inode: u64, + pub link: Option, +} + +/// What the walk skips or records by presence only. +#[derive(Clone, Debug, Default)] +pub struct SnapshotRules { + /// Directory names skipped anywhere. + pub excluded: Vec, + /// Directory names skipped only at the project root. + pub excluded_root: Vec, + /// File name suffixes recorded by presence only. + pub presence_suffixes: Vec, + /// Device and inode of files holding this command's output. + pub output_files: Vec<(u64, u64)>, +} diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml new file mode 100644 index 0000000000..c3b4179eca --- /dev/null +++ b/crates/sqlbuild-cli/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "sqlbuild-cli" +version.workspace = true +description = "The native `sqb` executable: reused compiles served without Python, every other command handed to the Python CLI." +edition.workspace = true +license-file = "../../LICENSE" +publish.workspace = true +rust-version.workspace = true + +[[bin]] +name = "sqb" +path = "src/bin/sqb.rs" + +[[bin]] +name = "sqlbuild" +path = "src/bin/sqlbuild.rs" + +[dependencies] +blake3.workspace = true +getrandom.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlbuild-cache.workspace = true +sqlbuild-core.workspace = true + +[target.'cfg(unix)'.dependencies] +rustix.workspace = true + +[target.'cfg(windows)'.dependencies] +ctrlc.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-cli/pyproject.toml b/crates/sqlbuild-cli/pyproject.toml new file mode 100644 index 0000000000..a1c14ed537 --- /dev/null +++ b/crates/sqlbuild-cli/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["maturin>=1.9,<2"] +build-backend = "maturin" + +[project] +name = "sqlbuild-cli" +dynamic = ["version"] +description = "The native sqb and sqlbuild executables of SQLBuild" +readme = "../../README.md" +license = "Apache-2.0" +requires-python = ">=3.12,<3.15" + +[tool.maturin] +bindings = "bin" + +[tool.uv] +cache-keys = [ + { file = "pyproject.toml" }, + { file = "Cargo.toml" }, + { file = "../../Cargo.toml" }, + { file = "../*/Cargo.toml" }, + { file = "../*/src/**/*.rs" }, +] diff --git a/crates/sqlbuild-cli/src/bin/sqb.rs b/crates/sqlbuild-cli/src/bin/sqb.rs new file mode 100644 index 0000000000..e3da9511e5 --- /dev/null +++ b/crates/sqlbuild-cli/src/bin/sqb.rs @@ -0,0 +1,9 @@ +//! `sqb`: the native SQLBuild command line. + +use std::process::ExitCode; + +use sqlbuild_cli::launcher::main::run_launcher::run_launcher; + +fn main() -> ExitCode { + run_launcher() +} diff --git a/crates/sqlbuild-cli/src/bin/sqlbuild.rs b/crates/sqlbuild-cli/src/bin/sqlbuild.rs new file mode 100644 index 0000000000..7a0aa9b7ae --- /dev/null +++ b/crates/sqlbuild-cli/src/bin/sqlbuild.rs @@ -0,0 +1,9 @@ +//! `sqlbuild`: the native SQLBuild command line, the same program as `sqb`. + +use std::process::ExitCode; + +use sqlbuild_cli::launcher::main::run_launcher::run_launcher; + +fn main() -> ExitCode { + run_launcher() +} diff --git a/crates/sqlbuild-cli/src/compute_logs/_helpers/files.rs b/crates/sqlbuild-cli/src/compute_logs/_helpers/files.rs new file mode 100644 index 0000000000..085ffc9abe --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/_helpers/files.rs @@ -0,0 +1,51 @@ +//! Durable file steps of a capture: exclusive creation, atomic metadata and directory syncs. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; + +use crate::compute_logs::constants::METADATA_FILE_NAME; + +/// Random 128-bit hex text, as `uuid.uuid4().hex`. +pub(crate) fn uuid4_hex() -> io::Result { + let mut bytes = [0_u8; 16]; + getrandom::fill(&mut bytes).map_err(|error| io::Error::other(error.to_string()))?; + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + Ok(bytes.iter().map(|byte| format!("{byte:02x}")).collect()) +} + +/// Open `path` for writing, failing when it exists, as Python's `open(path, "xb")`. +pub(crate) fn create_new(path: &Path) -> io::Result { + OpenOptions::new().write(true).create_new(true).open(path) +} + +/// `os.fsync` of a directory, which Python opens read-only. +pub(crate) fn sync_directory(path: &Path) -> io::Result<()> { + File::open(path)?.sync_all() +} + +/// Write `metadata.json` through a synced temporary file replaced into place. +pub(crate) fn write_metadata(directory: &Path, json: &str) -> io::Result<()> { + let temporary = directory.join(format!(".{METADATA_FILE_NAME}.{}.tmp", uuid4_hex()?)); + let result = write_and_replace(&temporary, &directory.join(METADATA_FILE_NAME), json) + .and_then(|()| sync_directory(directory)); + if temporary.exists() { + let _ = std::fs::remove_file(&temporary); + } + result +} + +fn write_and_replace(temporary: &Path, destination: &Path, json: &str) -> io::Result<()> { + let mut file = create_new(temporary)?; + file.write_all(json.as_bytes())?; + file.flush()?; + file.sync_all()?; + drop(file); + std::fs::rename(temporary, destination) +} + +/// Whether `path` is a symbolic link, without following it. +pub(crate) fn is_symlink(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|metadata| metadata.file_type().is_symlink()) +} diff --git a/crates/sqlbuild-cli/src/compute_logs/_helpers/metadata.rs b/crates/sqlbuild-cli/src/compute_logs/_helpers/metadata.rs new file mode 100644 index 0000000000..1ed3c70ad6 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/_helpers/metadata.rs @@ -0,0 +1,136 @@ +//! `metadata_to_json` and the subset of `metadata_from_json` validation pruning relies on. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +use crate::compute_logs::_helpers::timestamps::{ + epoch_microseconds, parse_utc_timestamp, utc_date, utc_timestamp, +}; +use crate::compute_logs::constants::COMPUTE_LOG_FORMAT_VERSION; +use crate::compute_logs::models::{CaptureEnd, CaptureStart}; + +/// The streams' final byte counts: stdout, stderr and diagnostics. +pub(crate) type ByteCounts = (usize, usize, usize); + +/// A stored capture's identity and the instant pruning orders it by. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct StoredCapture { + pub(crate) invocation_id: String, + pub(crate) capture_date: String, + /// `completed_at`, or `started_at` while incomplete, in microseconds. + pub(crate) order_instant: i64, +} + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn integer(value: i64) -> JsonValue { + JsonValue::Integer(JsonInteger::from(value)) +} + +fn count(value: usize) -> JsonValue { + integer(i64::try_from(value).unwrap_or(i64::MAX)) +} + +/// `metadata_to_json`: sorted keys, compact separators, non-ASCII kept, one trailing newline. +pub(crate) fn metadata_json( + (start, invocation_id): (&CaptureStart, &str), + end: Option<(&CaptureEnd, i64, ByteCounts)>, +) -> String { + let started_at = epoch_microseconds(start.started_at); + let mut fields: Vec<(String, JsonValue)> = vec![ + ( + "format_version".to_owned(), + integer(COMPUTE_LOG_FORMAT_VERSION), + ), + ("invocation_id".to_owned(), text(invocation_id)), + ("command".to_owned(), text(&start.command)), + ("project_dir".to_owned(), text(&start.project_dir)), + ("started_at".to_owned(), text(&utc_timestamp(started_at))), + ("capture_date".to_owned(), text(&utc_date(started_at))), + ( + "target".to_owned(), + start.target.as_deref().map_or(JsonValue::Null, text), + ), + ("run_id".to_owned(), JsonValue::Null), + ]; + if let Some((end, completed_at, (stdout_bytes, stderr_bytes, diagnostics_bytes))) = end { + fields.extend([ + ( + "completed_at".to_owned(), + text(&utc_timestamp(completed_at)), + ), + ("exit_code".to_owned(), integer(end.exit_code)), + ("stdout_bytes".to_owned(), count(stdout_bytes)), + ("stderr_bytes".to_owned(), count(stderr_bytes)), + ("diagnostics_bytes".to_owned(), count(diagnostics_bytes)), + ]); + } + let options = StdlibJsonOptions { + ensure_ascii: false, + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + let encoded = + dumps(&JsonValue::Object(fields), &JsonDialect::Stdlib(options)).unwrap_or_default(); + format!("{encoded}\n") +} + +fn string_field<'a>( + object: &'a serde_json::Map, + key: &str, +) -> Option<&'a str> { + object.get(key).and_then(serde_json::Value::as_str) +} + +/// The capture a stored `metadata.json` describes, or `None` where Python's validation rejects it. +pub(crate) fn stored_capture(json: &str) -> Option { + let Ok(serde_json::Value::Object(object)) = serde_json::from_str::(json) + else { + return None; + }; + if object + .get("format_version") + .and_then(serde_json::Value::as_i64) + != Some(COMPUTE_LOG_FORMAT_VERSION) + { + return None; + } + let invocation_id = string_field(&object, "invocation_id")?; + let capture_date = string_field(&object, "capture_date")?; + let command = string_field(&object, "command")?; + let project_dir = string_field(&object, "project_dir")?; + let started_at = parse_utc_timestamp(string_field(&object, "started_at")?)?; + if command.trim().is_empty() + || project_dir.trim().is_empty() + || utc_date(started_at) != capture_date + { + return None; + } + let order_instant = match object.get("completed_at") { + None => started_at, + Some(value) => { + let completed_at = parse_utc_timestamp(value.as_str()?)?; + let integers = [ + "exit_code", + "stdout_bytes", + "stderr_bytes", + "diagnostics_bytes", + ]; + if completed_at < started_at + || !integers + .iter() + .all(|key| object.get(*key).is_some_and(serde_json::Value::is_i64)) + { + return None; + } + completed_at + } + }; + Some(StoredCapture { + invocation_id: invocation_id.to_owned(), + capture_date: capture_date.to_owned(), + order_instant, + }) +} diff --git a/crates/sqlbuild-cli/src/compute_logs/_helpers/mod.rs b/crates/sqlbuild-cli/src/compute_logs/_helpers/mod.rs new file mode 100644 index 0000000000..cecf8b0763 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod files; +pub(crate) mod metadata; +pub(crate) mod prune; +pub(crate) mod timestamps; diff --git a/crates/sqlbuild-cli/src/compute_logs/_helpers/prune.rs b/crates/sqlbuild-cli/src/compute_logs/_helpers/prune.rs new file mode 100644 index 0000000000..33e47e0194 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/_helpers/prune.rs @@ -0,0 +1,125 @@ +//! `LocalFilesystemComputeLogStorage.prune`: keep the newest complete captures under a file lock. + +use std::fs::OpenOptions; +use std::path::{Path, PathBuf}; + +use crate::compute_logs::_helpers::files::is_symlink; +use crate::compute_logs::_helpers::metadata::{StoredCapture, stored_capture}; +use crate::compute_logs::constants::{ + CAPTURE_DATE_HYPHENS, CAPTURE_DATE_LENGTH, COMPLETE_FILE_NAME, METADATA_FILE_NAME, + PRUNE_LOCK_FILE_NAME, +}; + +/// Python's `\d{4}-\d{2}-\d{2}\Z` date directory name. +fn is_capture_date(name: &str) -> bool { + let bytes = name.as_bytes(); + bytes.len() == CAPTURE_DATE_LENGTH + && bytes.iter().enumerate().all(|(index, byte)| { + if CAPTURE_DATE_HYPHENS.contains(&index) { + *byte == b'-' + } else { + byte.is_ascii_digit() + } + }) +} + +fn sorted_entries(directory: &Path) -> Option> { + let Ok(listing) = std::fs::read_dir(directory) else { + return None; + }; + let mut entries: Vec = Vec::new(); + for entry in listing { + let Ok(entry) = entry else { + return None; + }; + entries.push(entry.path()); + } + entries.sort(); + Some(entries) +} + +fn file_name(path: &Path) -> String { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Every complete capture as Python's `inventory` lists it, or `None` where it raises. +fn complete_captures(root: &Path) -> Option> { + let mut complete: Vec<(StoredCapture, PathBuf)> = Vec::new(); + for date_directory in sorted_entries(root)? { + if !is_capture_date(&file_name(&date_directory)) { + continue; + } + if is_symlink(&date_directory) { + return None; + } + if !date_directory.is_dir() { + continue; + } + for capture_directory in sorted_entries(&date_directory)? { + let name = file_name(&capture_directory); + if name.starts_with('.') { + continue; + } + if is_symlink(&capture_directory) { + return None; + } + if !capture_directory.is_dir() { + continue; + } + let Ok(json) = std::fs::read_to_string(capture_directory.join(METADATA_FILE_NAME)) + else { + return None; + }; + let capture = stored_capture(&json)?; + if capture.invocation_id != name || capture.capture_date != file_name(&date_directory) { + return None; + } + if capture_directory.join(COMPLETE_FILE_NAME).is_file() { + complete.push((capture, capture_directory)); + } + } + } + Some(complete) +} + +/// Python `filelock`'s `fcntl.flock` exclusive lock, released when the file closes. +#[cfg(unix)] +fn lock_exclusive(file: &std::fs::File) -> std::io::Result<()> { + rustix::fs::flock(file, rustix::fs::FlockOperation::LockExclusive).map_err(std::io::Error::from) +} + +/// Captures are never published here, as in Python, whose directory `fsync` fails on Windows. +#[cfg(not(unix))] +fn lock_exclusive(_file: &std::fs::File) -> std::io::Result<()> { + Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) +} + +/// Delete all but the `retain` most recently completed captures; any fault leaves them all. +pub(crate) fn prune(root: &Path, retain: usize) { + let Ok(lock) = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(root.join(PRUNE_LOCK_FILE_NAME)) + else { + return; + }; + if lock_exclusive(&lock).is_err() { + return; + } + if let Some(mut complete) = complete_captures(root) { + complete.sort_by(|left, right| { + (right.0.order_instant, &right.0.invocation_id) + .cmp(&(left.0.order_instant, &left.0.invocation_id)) + }); + for (_, directory) in complete.iter().skip(retain) { + if directory.join(COMPLETE_FILE_NAME).is_file() { + let _ = std::fs::remove_dir_all(directory); + } + } + } + drop(lock); +} diff --git a/crates/sqlbuild-cli/src/compute_logs/_helpers/timestamps.rs b/crates/sqlbuild-cli/src/compute_logs/_helpers/timestamps.rs new file mode 100644 index 0000000000..ea16ce042f --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/_helpers/timestamps.rs @@ -0,0 +1,103 @@ +//! UTC timestamps as Python's `datetime.isoformat()` writes them, with `Z` for `+00:00`. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::compute_logs::constants::{ + DAYS_PER_ERA, EPOCH_DAY_OFFSET, FEBRUARY, MARCH_BASED_JANUARY, MICROSECOND_DIGITS, + MICROSECONDS_PER_SECOND, SECONDS_PER_DAY, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, YEARS_PER_ERA, +}; + +/// Microseconds since the Unix epoch of `instant`. +pub(crate) fn epoch_microseconds(instant: SystemTime) -> i64 { + match instant.duration_since(UNIX_EPOCH) { + Ok(elapsed) => i64::try_from(elapsed.as_micros()).unwrap_or(i64::MAX), + Err(_) => 0, + } +} + +/// The proleptic Gregorian `(year, month, day)` of days since 1970-01-01 (Hinnant's algorithm). +fn civil_date(days: i64) -> (i64, i64, i64) { + let shifted = days + EPOCH_DAY_OFFSET; + let era = shifted.div_euclid(DAYS_PER_ERA); + let day_of_era = shifted.rem_euclid(DAYS_PER_ERA); + let year_of_era = + (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = month_index + + if month_index < MARCH_BASED_JANUARY { + 3 + } else { + -9 + }; + let year = year_of_era + era * YEARS_PER_ERA + i64::from(month <= FEBRUARY); + (year, month, day) +} + +/// Days since 1970-01-01 of a proleptic Gregorian date (Hinnant's algorithm). +fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { + let shifted_year = year - i64::from(month <= FEBRUARY); + let era = shifted_year.div_euclid(YEARS_PER_ERA); + let year_of_era = shifted_year.rem_euclid(YEARS_PER_ERA); + let month_index = month + if month > FEBRUARY { -3 } else { 9 }; + let day_of_year = (153 * month_index + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * DAYS_PER_ERA + day_of_era - EPOCH_DAY_OFFSET +} + +/// `YYYY-MM-DD` of a UTC instant, as `capture_date`. +pub(crate) fn utc_date(microseconds: i64) -> String { + let (year, month, day) = + civil_date(microseconds.div_euclid(MICROSECONDS_PER_SECOND * SECONDS_PER_DAY)); + format!("{year:04}-{month:02}-{day:02}") +} + +/// `datetime.isoformat()` of a UTC instant with `+00:00` written as `Z`. +pub(crate) fn utc_timestamp(microseconds: i64) -> String { + let of_day = microseconds + .div_euclid(MICROSECONDS_PER_SECOND) + .rem_euclid(SECONDS_PER_DAY); + let fraction = microseconds.rem_euclid(MICROSECONDS_PER_SECOND); + let fraction_text = match fraction { + 0 => String::new(), + _ => format!(".{fraction:06}"), + }; + format!( + "{}T{:02}:{:02}:{:02}{fraction_text}Z", + utc_date(microseconds), + of_day / SECONDS_PER_HOUR, + of_day % SECONDS_PER_HOUR / SECONDS_PER_MINUTE, + of_day % SECONDS_PER_MINUTE + ) +} + +fn number(text: &str) -> Option { + let Ok(value) = text.parse::() else { + return None; + }; + Some(value) +} + +/// Microseconds of a `YYYY-MM-DDTHH:MM:SS[.ffffff]` UTC timestamp ending in `Z` or `+00:00`. +pub(crate) fn parse_utc_timestamp(text: &str) -> Option { + let body = text + .strip_suffix('Z') + .or_else(|| text.strip_suffix("+00:00"))?; + let (date, time) = body.split_once('T')?; + let (clock, fraction) = time.split_once('.').unwrap_or((time, "")); + let dates: Vec = date.split('-').map(number).collect::>()?; + let clocks: Vec = clock.split(':').map(number).collect::>()?; + let ([year, month, day], [hour, minute, second]) = (dates.as_slice(), clocks.as_slice()) else { + return None; + }; + let micros = match fraction { + "" => 0, + digits => number(format!("{digits:0<6}").get(..MICROSECOND_DIGITS)?)?, + }; + let seconds = days_from_civil(*year, *month, *day) * SECONDS_PER_DAY + + hour * SECONDS_PER_HOUR + + minute * SECONDS_PER_MINUTE + + second; + Some(seconds * MICROSECONDS_PER_SECOND + micros) +} diff --git a/crates/sqlbuild-cli/src/compute_logs/constants.rs b/crates/sqlbuild-cli/src/compute_logs/constants.rs new file mode 100644 index 0000000000..c0ee017821 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/constants.rs @@ -0,0 +1,29 @@ +//! Python's `sqlbuild.runtime.compute_logs.constants` and storage file names. + +pub const COMPUTE_LOG_FORMAT_VERSION: i64 = 1; +pub const DEFAULT_RETENTION_COUNT: usize = 20; +pub const METADATA_FILE_NAME: &str = "metadata.json"; +pub const COMPLETE_FILE_NAME: &str = "complete"; +pub const PRUNE_LOCK_FILE_NAME: &str = ".prune.lock"; +pub const STDOUT_FILE_NAME: &str = "stdout.log"; +pub const STDERR_FILE_NAME: &str = "stderr.log"; +pub const DIAGNOSTICS_FILE_NAME: &str = "diagnostics.jsonl"; +/// Stream files in the order Python creates them. +pub const STREAM_FILE_NAMES: [&str; 3] = + [STDOUT_FILE_NAME, STDERR_FILE_NAME, DIAGNOSTICS_FILE_NAME]; +pub const MICROSECONDS_PER_SECOND: i64 = 1_000_000; +pub const SECONDS_PER_DAY: i64 = 86_400; +pub const SECONDS_PER_HOUR: i64 = 3_600; +pub const SECONDS_PER_MINUTE: i64 = 60; +pub const MICROSECOND_DIGITS: usize = 6; +/// Days in a 400-year Gregorian era. +pub const DAYS_PER_ERA: i64 = 146_097; +pub const YEARS_PER_ERA: i64 = 400; +pub const FEBRUARY: i64 = 2; +/// January's index when months count from March, as the civil date algorithm does. +pub const MARCH_BASED_JANUARY: i64 = 10; +/// Days from 0000-03-01 to 1970-01-01. +pub const EPOCH_DAY_OFFSET: i64 = 719_468; +/// `YYYY-MM-DD`: the length of a capture date directory name and its hyphen positions. +pub const CAPTURE_DATE_LENGTH: usize = 10; +pub const CAPTURE_DATE_HYPHENS: [usize; 2] = [4, 7]; diff --git a/crates/sqlbuild-cli/src/compute_logs/main/finish_capture.rs b/crates/sqlbuild-cli/src/compute_logs/main/finish_capture.rs new file mode 100644 index 0000000000..efc79f70b1 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/main/finish_capture.rs @@ -0,0 +1,46 @@ +//! Append a command's output, publish its final metadata and prune, as Python's finalize does. + +use std::io::{self, Write}; +use std::time::SystemTime; + +use crate::compute_logs::_helpers::files::{create_new, sync_directory, write_metadata}; +use crate::compute_logs::_helpers::metadata::metadata_json; +use crate::compute_logs::_helpers::prune::prune; +use crate::compute_logs::_helpers::timestamps::epoch_microseconds; +use crate::compute_logs::constants::{COMPLETE_FILE_NAME, DEFAULT_RETENTION_COUNT}; +use crate::compute_logs::models::{CaptureEnd, OpenCapture}; + +/// Complete the capture; a failure leaves it incomplete, as Python's best-effort capture does. +pub fn finish_capture(capture: OpenCapture, end: &CaptureEnd) -> io::Result<()> { + let OpenCapture { + start, + invocation_id, + directory, + mut streams, + } = capture; + let [stdout, stderr, diagnostics] = streams.as_mut_slice() else { + return Err(io::Error::other("capture streams are not open")); + }; + stdout.write_all(&end.stdout)?; + stderr.write_all(&end.stderr)?; + for stream in [&mut *stdout, &mut *stderr, &mut *diagnostics] { + stream.flush()?; + stream.sync_all()?; + } + drop(streams); + write_metadata( + &directory, + &metadata_json( + (&start, &invocation_id), + Some(( + end, + epoch_microseconds(SystemTime::now()), + (end.stdout.len(), end.stderr.len(), 0), + )), + ), + )?; + create_new(&directory.join(COMPLETE_FILE_NAME))?.sync_all()?; + sync_directory(&directory)?; + prune(&start.root, DEFAULT_RETENTION_COUNT); + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/compute_logs/main/mod.rs b/crates/sqlbuild-cli/src/compute_logs/main/mod.rs new file mode 100644 index 0000000000..008c6b31d2 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/main/mod.rs @@ -0,0 +1,2 @@ +pub mod finish_capture; +pub mod start_capture; diff --git a/crates/sqlbuild-cli/src/compute_logs/main/start_capture.rs b/crates/sqlbuild-cli/src/compute_logs/main/start_capture.rs new file mode 100644 index 0000000000..35cf8ab225 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/main/start_capture.rs @@ -0,0 +1,71 @@ +//! Publish an incomplete capture before the command writes output, as `start_capture` does. + +use std::io::{self, Write}; +use std::path::Path; + +use crate::compute_logs::_helpers::files::{ + create_new, is_symlink, sync_directory, uuid4_hex, write_metadata, +}; +use crate::compute_logs::_helpers::metadata::metadata_json; +use crate::compute_logs::_helpers::timestamps::{epoch_microseconds, utc_date}; +use crate::compute_logs::constants::STREAM_FILE_NAMES; +use crate::compute_logs::models::{CaptureStart, OpenCapture}; + +/// The published capture, or the failure that leaves the command running without one. +pub fn start_capture(start: CaptureStart) -> io::Result { + if is_symlink(&start.root) { + return Err(io::Error::other( + "configured compute log root cannot be a symlink", + )); + } + std::fs::create_dir_all(&start.root)?; + let date_directory = start + .root + .join(utc_date(epoch_microseconds(start.started_at))); + if is_symlink(&date_directory) { + return Err(io::Error::other("compute log path cannot be a symlink")); + } + std::fs::create_dir_all(&date_directory)?; + let invocation_id = uuid4_hex()?; + let directory = date_directory.join(&invocation_id); + if directory.exists() || is_symlink(&directory) { + return Err(io::Error::other("capture already exists")); + } + let staging = date_directory.join(format!(".{invocation_id}.{}.tmp", uuid4_hex()?)); + std::fs::create_dir(&staging)?; + match published((&start, &invocation_id), &staging, &directory) { + Ok(streams) => Ok(OpenCapture { + start, + invocation_id, + directory, + streams, + }), + Err(error) => { + let _ = std::fs::remove_dir_all(&staging); + let _ = std::fs::remove_dir_all(&directory); + Err(error) + } + } +} + +fn published( + start: (&CaptureStart, &str), + staging: &Path, + directory: &Path, +) -> io::Result> { + let mut streams: Vec = Vec::new(); + for name in STREAM_FILE_NAMES { + streams.push(create_new(&staging.join(name))?); + } + write_metadata(staging, &metadata_json(start, None))?; + for stream in &mut streams { + stream.flush()?; + stream.sync_all()?; + } + sync_directory(staging)?; + std::fs::rename(staging, directory)?; + if let Some(parent) = directory.parent() { + sync_directory(parent)?; + } + Ok(streams) +} diff --git a/crates/sqlbuild-cli/src/compute_logs/mod.rs b/crates/sqlbuild-cli/src/compute_logs/mod.rs new file mode 100644 index 0000000000..7366aa64c1 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/mod.rs @@ -0,0 +1,9 @@ +//! The project-local compute log capture Python's `LocalFilesystemComputeLogStorage` writes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/compute_logs/models.rs b/crates/sqlbuild-cli/src/compute_logs/models.rs new file mode 100644 index 0000000000..9511254d2e --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/models.rs @@ -0,0 +1,35 @@ +//! A compute log capture's identity and the metadata it publishes. + +use std::fs::File; +use std::path::PathBuf; +use std::time::SystemTime; + +/// What one capture records before its command runs, as Python's `CaptureMetadata`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CaptureStart { + /// The resolved compute log root, `/logs`. + pub root: PathBuf, + pub command: String, + pub project_dir: String, + pub target: Option, + pub started_at: SystemTime, +} + +/// What the finished command adds, as Python's `FinalCaptureMetadata`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CaptureEnd { + pub stdout: Vec, + pub stderr: Vec, + pub exit_code: i64, +} + +/// A published, incomplete capture and its open stream files. +#[derive(Debug)] +pub struct OpenCapture { + pub start: CaptureStart, + /// The invocation id naming the capture directory, as `uuid4().hex`. + pub invocation_id: String, + pub directory: PathBuf, + /// `stdout.log`, `stderr.log` and `diagnostics.jsonl`. + pub streams: Vec, +} diff --git a/crates/sqlbuild-cli/src/compute_logs/tests.rs b/crates/sqlbuild-cli/src/compute_logs/tests.rs new file mode 100644 index 0000000000..6677eaa86e --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_capture.rs"] +mod capture; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_metadata.rs"] +mod metadata; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/compute_logs/tests/helpers.rs b/crates/sqlbuild-cli/src/compute_logs/tests/helpers.rs new file mode 100644 index 0000000000..bcfff032a4 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/tests/helpers.rs @@ -0,0 +1,28 @@ +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use crate::compute_logs::models::CaptureStart; + +pub(crate) fn capture_start(root: &Path, offset_seconds: u64) -> CaptureStart { + CaptureStart { + root: root.to_path_buf(), + command: "compile".to_owned(), + project_dir: "/project".to_owned(), + target: None, + started_at: SystemTime::now() - Duration::from_secs(offset_seconds), + } +} + +pub(crate) fn complete_captures(root: &Path) -> Vec { + let mut found: Vec = Vec::new(); + for date in std::fs::read_dir(root).into_iter().flatten().flatten() { + for capture in std::fs::read_dir(date.path()) + .into_iter() + .flatten() + .flatten() + { + found.extend(Some(capture.path()).filter(|path| path.join("complete").is_file())); + } + } + found +} diff --git a/crates/sqlbuild-cli/src/compute_logs/tests/test_capture.rs b/crates/sqlbuild-cli/src/compute_logs/tests/test_capture.rs new file mode 100644 index 0000000000..5a2322ffed --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/tests/test_capture.rs @@ -0,0 +1,57 @@ +use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; +use crate::compute_logs::main::finish_capture::finish_capture; +use crate::compute_logs::main::start_capture::start_capture; +use crate::compute_logs::models::CaptureEnd; +use crate::compute_logs::tests::helpers::{capture_start, complete_captures}; +use crate::compute_logs::tests::test_types::PruneTestCase; + +#[test] +fn given_finished_command_when_capturing_then_streams_metadata_and_pruning_match_python() { + let test_cases = [ + PruneTestCase { + description: "below the retention count every capture stays", + existing: 2, + expected_complete: 3, + }, + PruneTestCase { + description: "the oldest captures beyond the retention count are deleted", + existing: DEFAULT_RETENTION_COUNT + 2, + expected_complete: DEFAULT_RETENTION_COUNT, + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary log root"); + let end = CaptureEnd { + stdout: b"Compile ready\n".to_vec(), + stderr: b"Inputs unchanged\n".to_vec(), + exit_code: 0, + }; + for offset in 0..test_case.existing { + let capture = start_capture(capture_start(root.path(), 60 + offset as u64)) + .expect("capture starts"); + finish_capture(capture, &end).expect("capture finishes"); + } + + let capture = start_capture(capture_start(root.path(), 0)).expect("capture starts"); + let directory = capture.directory.clone(); + finish_capture(capture, &end).expect("capture finishes"); + + assert_eq!( + ( + std::fs::read(directory.join("stdout.log")).expect("stdout log"), + std::fs::read(directory.join("stderr.log")).expect("stderr log"), + std::fs::read(directory.join("diagnostics.jsonl")).expect("diagnostics log"), + complete_captures(root.path()).len(), + ), + ( + end.stdout.clone(), + end.stderr.clone(), + Vec::new(), + test_case.expected_complete, + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/compute_logs/tests/test_metadata.rs b/crates/sqlbuild-cli/src/compute_logs/tests/test_metadata.rs new file mode 100644 index 0000000000..94e63b1df9 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/tests/test_metadata.rs @@ -0,0 +1,93 @@ +use std::path::PathBuf; +use std::time::{Duration, UNIX_EPOCH}; + +use crate::compute_logs::_helpers::metadata::metadata_json; +use crate::compute_logs::_helpers::timestamps::{parse_utc_timestamp, utc_timestamp}; +use crate::compute_logs::models::{CaptureEnd, CaptureStart}; +use crate::compute_logs::tests::test_types::{MetadataTestCase, TimestampTestCase}; + +const STARTED_MICROSECONDS: u64 = 1_791_732_481_125_950; + +#[test] +fn given_utc_instant_when_formatting_then_matches_python_isoformat_and_parses_back() { + let test_cases = [ + TimestampTestCase { + description: "microseconds are written with six digits", + microseconds: 1_791_732_481_125_950, + expected_timestamp: "2026-10-11T15:28:01.125950Z", + }, + TimestampTestCase { + description: "a whole second omits the fraction", + microseconds: 1_791_732_482_000_000, + expected_timestamp: "2026-10-11T15:28:02Z", + }, + TimestampTestCase { + description: "a leap day keeps one microsecond", + microseconds: 951_782_400_000_001, + expected_timestamp: "2000-02-29T00:00:00.000001Z", + }, + TimestampTestCase { + description: "an instant before the epoch", + microseconds: -1_000_000, + expected_timestamp: "1969-12-31T23:59:59Z", + }, + ]; + + for test_case in test_cases { + let timestamp = utc_timestamp(test_case.microseconds); + + assert_eq!( + (timestamp.as_str(), parse_utc_timestamp(×tamp)), + (test_case.expected_timestamp, Some(test_case.microseconds)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_capture_when_writing_metadata_then_bytes_match_python_metadata_to_json() { + let test_cases = [ + MetadataTestCase { + description: "an open capture", + target: None, + end: None, + expected_json: "{\"capture_date\":\"2026-10-11\",\"command\":\"compile\",\"format_version\":1,\"invocation_id\":\"abc\",\"project_dir\":\"/p/\u{e9}\",\"run_id\":null,\"started_at\":\"2026-10-11T15:28:01.125950Z\",\"target\":null}\n", + }, + MetadataTestCase { + description: "a completed capture", + target: Some("dev"), + end: Some((1_791_732_482, 1, 3, 4)), + expected_json: "{\"capture_date\":\"2026-10-11\",\"command\":\"compile\",\"completed_at\":\"2026-10-11T15:28:02Z\",\"diagnostics_bytes\":0,\"exit_code\":1,\"format_version\":1,\"invocation_id\":\"abc\",\"project_dir\":\"/p/\u{e9}\",\"run_id\":null,\"started_at\":\"2026-10-11T15:28:01.125950Z\",\"stderr_bytes\":4,\"stdout_bytes\":3,\"target\":\"dev\"}\n", + }, + ]; + + for test_case in test_cases { + let start = CaptureStart { + root: PathBuf::from("/p/logs"), + command: "compile".to_owned(), + project_dir: "/p/\u{e9}".to_owned(), + target: test_case.target.map(str::to_owned), + started_at: UNIX_EPOCH + Duration::from_micros(STARTED_MICROSECONDS), + }; + let end = test_case.end.map(|(completed, exit_code, stdout, stderr)| { + ( + CaptureEnd { + stdout: vec![0; stdout], + stderr: vec![0; stderr], + exit_code, + }, + completed * 1_000_000, + (stdout, stderr, 0), + ) + }); + + let json = metadata_json( + (&start, "abc"), + end.as_ref() + .map(|(end, completed, counts)| (end, *completed, *counts)), + ); + + assert_eq!(json, test_case.expected_json, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cli/src/compute_logs/tests/test_types.rs b/crates/sqlbuild-cli/src/compute_logs/tests/test_types.rs new file mode 100644 index 0000000000..7f41b10a28 --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/tests/test_types.rs @@ -0,0 +1,22 @@ +/// One UTC instant and the text Python's `isoformat()` gives it. +pub(crate) struct TimestampTestCase { + pub(crate) description: &'static str, + pub(crate) microseconds: i64, + pub(crate) expected_timestamp: &'static str, +} + +/// One capture's metadata and the exact `metadata_to_json` text. +pub(crate) struct MetadataTestCase { + pub(crate) description: &'static str, + pub(crate) target: Option<&'static str>, + /// `(completed_at seconds, exit code, stdout bytes, stderr bytes)` once complete. + pub(crate) end: Option<(i64, i64, usize, usize)>, + pub(crate) expected_json: &'static str, +} + +/// Completed captures already in the log root and how many pruning keeps. +pub(crate) struct PruneTestCase { + pub(crate) description: &'static str, + pub(crate) existing: usize, + pub(crate) expected_complete: usize, +} diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs new file mode 100644 index 0000000000..c8c85f8e54 --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -0,0 +1,56 @@ +//! Decide how one invocation runs: natively, from a launch record, or in the Python CLI. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::time::{Instant, SystemTime}; + +use crate::launcher::_helpers::handoff::{hand_off, python_cli}; +use crate::launcher::_helpers::launch_key::{launch_key, launch_record_path}; +use crate::launcher::_helpers::messages::{print_version, report_missing_python_cli}; +use crate::launcher::constants::{ + HANDOFF_FAILURE_EXIT_CODE, LAUNCH_RECORD_OPTION, VERSION_OPTIONS, +}; +use crate::replay::main::serve_reused_compile::serve_reused_compile; + +/// The launch record path and key for these arguments, when the directory is known. +fn launch_record( + program: &Path, + script: &Path, + arguments: &[OsString], +) -> Option<(PathBuf, String)> { + let Ok(cwd) = std::env::current_dir() else { + return None; + }; + let key = launch_key(program, script, arguments, &cwd); + launch_record_path(arguments, &cwd, &key).map(|path| (path, key)) +} + +fn exit_code(code: i64) -> i32 { + i32::try_from(code).unwrap_or(HANDOFF_FAILURE_EXIT_CODE) +} + +/// Run one invocation started at `started` and return its exit code. +pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) -> i32 { + if let [only] = arguments.as_slice() + && VERSION_OPTIONS.iter().any(|option| only == option) + { + print_version(); + return 0; + } + let program: PathBuf = std::env::current_exe().unwrap_or_default(); + let Some(script) = python_cli(&program) else { + report_missing_python_cli(&program); + return HANDOFF_FAILURE_EXIT_CODE; + }; + let mut handed: Vec = Vec::with_capacity(arguments.len() + 1); + if let Some((path, key)) = launch_record(&program, &script, &arguments) { + if let Some(code) = serve_reused_compile(&path, &key, started) { + return exit_code(code); + } + let mut option = OsString::from(LAUNCH_RECORD_OPTION); + option.push(path.as_os_str()); + handed.push(option); + } + handed.extend(arguments); + hand_off(&script, &handed) +} diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/handoff.rs b/crates/sqlbuild-cli/src/launcher/_helpers/handoff.rs new file mode 100644 index 0000000000..d6c001178b --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/_helpers/handoff.rs @@ -0,0 +1,42 @@ +//! Find the Python CLI script installed beside this program and run it with the same arguments. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::launcher::_helpers::messages::report_handoff_failure; +use crate::launcher::constants::{HANDOFF_FAILURE_EXIT_CODE, PYTHON_CLI_SCRIPT}; + +/// The Python CLI script beside this program or beside the file it links to. +pub(crate) fn python_cli(program: &Path) -> Option { + let resolved = std::fs::canonicalize(program).unwrap_or_else(|_| program.to_path_buf()); + [program, resolved.as_path()] + .into_iter() + .filter_map(Path::parent) + .map(|directory| directory.join(PYTHON_CLI_SCRIPT)) + .find(|candidate| candidate.is_file()) +} + +fn failure(script: &Path, error: &std::io::Error) -> i32 { + report_handoff_failure(script, error); + HANDOFF_FAILURE_EXIT_CODE +} + +/// Replace this process with the Python CLI; returns only when it cannot start. +#[cfg(unix)] +pub(crate) fn hand_off(script: &Path, arguments: &[OsString]) -> i32 { + use std::os::unix::process::CommandExt; + + let error = Command::new(script).args(arguments).exec(); + failure(script, &error) +} + +/// Run the Python CLI as a child that alone handles Ctrl+C, and return its exit code. +#[cfg(not(unix))] +pub(crate) fn hand_off(script: &Path, arguments: &[OsString]) -> i32 { + let _ = ctrlc::set_handler(|| {}); + match Command::new(script).args(arguments).status() { + Ok(status) => status.code().unwrap_or(HANDOFF_FAILURE_EXIT_CODE), + Err(error) => failure(script, &error), + } +} diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/launch_key.rs b/crates/sqlbuild-cli/src/launcher/_helpers/launch_key.rs new file mode 100644 index 0000000000..1ef154b517 --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/_helpers/launch_key.rs @@ -0,0 +1,127 @@ +//! The launch key: a digest of every process fact the Python CLI's reuse identity is built from. + +use std::ffi::OsString; +use std::io::IsTerminal; +use std::path::{Path, PathBuf}; +use std::time::UNIX_EPOCH; + +use crate::launcher::constants::{ + LAUNCH_KEY_DOMAIN, MISSING_FILE_STAMP, PROJECT_DIR_ABBREVIATION_PREFIXES, PROJECT_DIR_OPTIONS, + VOLATILE_ENVIRONMENT, +}; +use crate::replay::constants::{LAUNCH_RECORD_DIRECTORY, LAUNCH_RECORD_SUFFIX}; + +#[cfg(unix)] +fn os_bytes(value: &std::ffi::OsStr) -> Vec { + use std::os::unix::ffi::OsStrExt; + value.as_bytes().to_vec() +} + +#[cfg(not(unix))] +fn os_bytes(value: &std::ffi::OsStr) -> Vec { + value.to_string_lossy().into_owned().into_bytes() +} + +fn update(hasher: &mut blake3::Hasher, field: &[u8]) { + let _ = hasher.update(&(field.len() as u64).to_le_bytes()); + let _ = hasher.update(field); +} + +fn file_stamp(path: &Path) -> Vec { + let Ok(metadata) = std::fs::metadata(path) else { + return MISSING_FILE_STAMP.to_vec(); + }; + let modified: u128 = match metadata + .modified() + .map(|time| time.duration_since(UNIX_EPOCH)) + { + Ok(Ok(elapsed)) => elapsed.as_nanos(), + _ => 0, + }; + format!("{}:{modified}", metadata.len()).into_bytes() +} + +fn is_volatile(name: &std::ffi::OsStr) -> bool { + VOLATILE_ENVIRONMENT + .iter() + .any(|volatile| name == *volatile) +} + +/// The digest of the program, its Python handoff, arguments, directory, environment and terminals. +pub(crate) fn launch_key( + program: &Path, + python_cli: &Path, + arguments: &[OsString], + cwd: &Path, +) -> String { + let mut hasher = blake3::Hasher::new(); + update(&mut hasher, LAUNCH_KEY_DOMAIN.as_bytes()); + update(&mut hasher, env!("CARGO_PKG_VERSION").as_bytes()); + for path in [program, python_cli] { + update(&mut hasher, &os_bytes(path.as_os_str())); + update(&mut hasher, &file_stamp(path)); + } + if let Ok(script) = std::fs::read(python_cli) { + update(&mut hasher, &script); + } + update(&mut hasher, &(arguments.len() as u64).to_le_bytes()); + for argument in arguments { + update(&mut hasher, &os_bytes(argument)); + } + update(&mut hasher, &os_bytes(cwd.as_os_str())); + let mut environment: Vec<(OsString, OsString)> = std::env::vars_os() + .filter(|(name, _)| !is_volatile(name)) + .collect(); + environment.sort(); + for (name, value) in &environment { + update(&mut hasher, &os_bytes(name)); + update(&mut hasher, &os_bytes(value)); + } + let terminals = [ + std::io::stdin().is_terminal(), + std::io::stdout().is_terminal(), + std::io::stderr().is_terminal(), + ]; + update(&mut hasher, &terminals.map(u8::from)); + hasher.finalize().to_hex().to_string() +} + +/// The project directory the arguments name, or `None` when argparse might read them otherwise. +fn named_project_dir(arguments: &[OsString]) -> Option> { + let mut named: Option = None; + let mut index = 0; + while index < arguments.len() { + let text = arguments[index].to_string_lossy(); + if PROJECT_DIR_OPTIONS.contains(&text.as_ref()) { + named = Some(arguments.get(index + 1)?.clone()); + index += 2; + continue; + } + if let Some((option, value)) = text.split_once('=') + && PROJECT_DIR_OPTIONS.contains(&option) + { + named = Some(OsString::from(value)); + } else if PROJECT_DIR_ABBREVIATION_PREFIXES + .iter() + .any(|prefix| text.starts_with(prefix)) + { + return None; + } + index += 1; + } + Some(named) +} + +/// Where the Python CLI keeps this invocation's launch record, if the project directory is plain. +pub(crate) fn launch_record_path(arguments: &[OsString], cwd: &Path, key: &str) -> Option { + let project = match named_project_dir(arguments)? { + Some(named) => cwd.join(named), + None => cwd.to_path_buf(), + }; + let mut path = project; + for part in LAUNCH_RECORD_DIRECTORY { + path.push(part); + } + path.push(format!("{key}{LAUNCH_RECORD_SUFFIX}")); + Some(path) +} diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs b/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs new file mode 100644 index 0000000000..8a91e6387d --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs @@ -0,0 +1,29 @@ +//! The launcher's own lines on the standard streams, written as the Python CLI writes text. + +use std::io::{self, Write}; +use std::path::Path; + +use crate::launcher::constants::{PROGRAM_NAME, PYTHON_CLI_SCRIPT}; + +/// argparse's `--version` line: `sqb ` on stdout. +pub(crate) fn print_version() { + let _ = writeln!(io::stdout(), "{PROGRAM_NAME} {}", env!("CARGO_PKG_VERSION")); +} + +/// The Python CLI script is not installed beside this program. +pub(crate) fn report_missing_python_cli(program: &Path) { + let _ = writeln!( + io::stderr(), + "{PROGRAM_NAME}: cannot find the SQLBuild Python CLI ({PYTHON_CLI_SCRIPT}) beside {}", + program.display() + ); +} + +/// The Python CLI script exists but could not be started. +pub(crate) fn report_handoff_failure(script: &Path, error: &io::Error) { + let _ = writeln!( + io::stderr(), + "{PROGRAM_NAME}: cannot start the SQLBuild Python CLI at {}: {error}", + script.display() + ); +} diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/mod.rs b/crates/sqlbuild-cli/src/launcher/_helpers/mod.rs new file mode 100644 index 0000000000..cbeb91cd81 --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod dispatch; +pub(crate) mod handoff; +pub(crate) mod launch_key; +pub(crate) mod messages; diff --git a/crates/sqlbuild-cli/src/launcher/constants.rs b/crates/sqlbuild-cli/src/launcher/constants.rs new file mode 100644 index 0000000000..82cad78763 --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/constants.rs @@ -0,0 +1,25 @@ +//! Names the launcher shares with the Python CLI and its packaging. + +/// The console script the `sqlbuild` wheel installs beside `sqb`; it runs the Python CLI. +#[cfg(not(windows))] +pub const PYTHON_CLI_SCRIPT: &str = "sqlbuild-python"; +/// The console script the `sqlbuild` wheel installs beside `sqb`; it runs the Python CLI. +#[cfg(windows)] +pub const PYTHON_CLI_SCRIPT: &str = "sqlbuild-python.exe"; +/// The first argument naming the launch record the Python CLI may write for this invocation. +pub const LAUNCH_RECORD_OPTION: &str = "--sqlbuild-launch-record="; +/// The program name argparse reports, fixed by the Python parser. +pub const PROGRAM_NAME: &str = "sqb"; +pub const VERSION_OPTIONS: [&str; 2] = ["--version", "-V"]; +/// Global options naming the project directory, exactly as the Python parser spells them. +pub const PROJECT_DIR_OPTIONS: [&str; 2] = ["--project-dir", "--sqb-project-dir"]; +/// Prefixes argparse could expand to a project directory option. +pub const PROJECT_DIR_ABBREVIATION_PREFIXES: [&str; 2] = ["--p", "--sq"]; +/// Variables whose value changes between otherwise identical shell commands. +pub const VOLATILE_ENVIRONMENT: [&str; 4] = ["OLDPWD", "PWD", "SHLVL", "_"]; +/// Domain separator of the launch key digest; bump when its inputs change. +pub const LAUNCH_KEY_DOMAIN: &str = "sqlbuild-launch-key-1"; +/// The launch key's stamp of a file that does not exist. +pub const MISSING_FILE_STAMP: &[u8] = b"missing"; +/// Exit code when the Python CLI cannot be started. +pub const HANDOFF_FAILURE_EXIT_CODE: i32 = 1; diff --git a/crates/sqlbuild-cli/src/launcher/main/mod.rs b/crates/sqlbuild-cli/src/launcher/main/mod.rs new file mode 100644 index 0000000000..092024d70e --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/main/mod.rs @@ -0,0 +1 @@ +pub mod run_launcher; diff --git a/crates/sqlbuild-cli/src/launcher/main/run_launcher.rs b/crates/sqlbuild-cli/src/launcher/main/run_launcher.rs new file mode 100644 index 0000000000..cc1f26c7b2 --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/main/run_launcher.rs @@ -0,0 +1,18 @@ +//! The `sqb` and `sqlbuild` executables' whole process. + +use std::ffi::OsString; +use std::process::ExitCode; +use std::time::{Instant, SystemTime}; + +use crate::launcher::_helpers::dispatch::launch; + +/// Run this process's invocation; an exit code beyond one byte ends the process directly. +pub fn run_launcher() -> ExitCode { + let started = (Instant::now(), SystemTime::now()); + let arguments: Vec = std::env::args_os().skip(1).collect(); + let code: i32 = launch(arguments, started); + match u8::try_from(code) { + Ok(code) => ExitCode::from(code), + Err(_) => std::process::exit(code), + } +} diff --git a/crates/sqlbuild-cli/src/launcher/mod.rs b/crates/sqlbuild-cli/src/launcher/mod.rs new file mode 100644 index 0000000000..e9e2755531 --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/mod.rs @@ -0,0 +1,8 @@ +//! The `sqb` process: answer `--version`, serve reused compiles natively, hand the rest to Python. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/launcher/tests.rs b/crates/sqlbuild-cli/src/launcher/tests.rs new file mode 100644 index 0000000000..41790fa54d --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_launch_key.rs"] +mod launch_key; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/launcher/tests/test_launch_key.rs b/crates/sqlbuild-cli/src/launcher/tests/test_launch_key.rs new file mode 100644 index 0000000000..49cd9caa28 --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/tests/test_launch_key.rs @@ -0,0 +1,100 @@ +use std::ffi::OsString; +use std::path::{Path, PathBuf}; + +use crate::launcher::_helpers::launch_key::{launch_key, launch_record_path}; +use crate::launcher::tests::test_types::{LaunchKeyTestCase, LaunchRecordPathTestCase}; + +#[test] +fn given_arguments_when_locating_launch_record_then_only_exact_project_options_count() { + let test_cases = [ + LaunchRecordPathTestCase { + description: "no project option uses the working directory", + arguments: &["compile", "--json"], + expected_project: Some("/work"), + }, + LaunchRecordPathTestCase { + description: "the last exact option wins, with or without an equals sign", + arguments: &[ + "--project-dir", + "first", + "compile", + "--sqb-project-dir=/shop", + ], + expected_project: Some("/shop"), + }, + LaunchRecordPathTestCase { + description: "a relative project joins the working directory", + arguments: &["compile", "--project-dir", "shop"], + expected_project: Some("/work/shop"), + }, + LaunchRecordPathTestCase { + description: "an abbreviation argparse would expand is not guessed", + arguments: &["compile", "--project", "shop"], + expected_project: None, + }, + LaunchRecordPathTestCase { + description: "a project option without its value is not guessed", + arguments: &["compile", "--project-dir"], + expected_project: None, + }, + ]; + + for test_case in test_cases { + let arguments: Vec = test_case.arguments.iter().map(OsString::from).collect(); + + let path = launch_record_path(&arguments, Path::new("/work"), "key"); + + assert_eq!( + path, + test_case + .expected_project + .map(|project| PathBuf::from(project).join("target/cache/launcher-v1/key.json")), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_changed_arguments_or_directory_when_keying_then_the_launch_key_changes() { + let test_cases = [ + LaunchKeyTestCase { + description: "the same invocation shares its key", + arguments: &["compile"], + cwd: "/work", + expected_same: true, + }, + LaunchKeyTestCase { + description: "another argument changes the key", + arguments: &["compile", "--json"], + cwd: "/work", + expected_same: false, + }, + LaunchKeyTestCase { + description: "another working directory changes the key", + arguments: &["compile"], + cwd: "/elsewhere", + expected_same: false, + }, + ]; + let program = std::env::current_exe().expect("test program"); + let base = launch_key( + &program, + &program, + &[OsString::from("compile")], + Path::new("/work"), + ); + + for test_case in test_cases { + let arguments: Vec = test_case.arguments.iter().map(OsString::from).collect(); + + let key = launch_key(&program, &program, &arguments, Path::new(test_case.cwd)); + + assert_eq!( + key == base, + test_case.expected_same, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/launcher/tests/test_types.rs b/crates/sqlbuild-cli/src/launcher/tests/test_types.rs new file mode 100644 index 0000000000..70da2158ed --- /dev/null +++ b/crates/sqlbuild-cli/src/launcher/tests/test_types.rs @@ -0,0 +1,14 @@ +/// Command-line arguments and where the launch record for them is looked up. +pub(crate) struct LaunchRecordPathTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) expected_project: Option<&'static str>, +} + +/// One invocation keyed beside `sqb compile` in `/work`. +pub(crate) struct LaunchKeyTestCase { + pub(crate) description: &'static str, + pub(crate) arguments: &'static [&'static str], + pub(crate) cwd: &'static str, + pub(crate) expected_same: bool, +} diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs new file mode 100644 index 0000000000..c06ee4903e --- /dev/null +++ b/crates/sqlbuild-cli/src/lib.rs @@ -0,0 +1,7 @@ +//! The native `sqb` executable: replays reused compiles and hands other invocations to Python. + +#![forbid(unsafe_code)] + +pub mod compute_logs; +pub mod launcher; +pub mod replay; diff --git a/crates/sqlbuild-cli/src/replay/_helpers/console.rs b/crates/sqlbuild-cli/src/replay/_helpers/console.rs new file mode 100644 index 0000000000..f2732b2bf1 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/_helpers/console.rs @@ -0,0 +1,57 @@ +//! The process's standard streams as the Python CLI writes them. + +use std::io::{self, Write}; + +/// Python's `f"{milliseconds / 1000:.1f}"`; both round the binary value half to even. +pub(crate) fn tenths_of_seconds(milliseconds: i64) -> String { + let seconds = milliseconds as f64 / 1000.0; + format!("{seconds:.1}") +} + +/// Text as a Python text stream writes it: `\n` becomes the platform line separator. +pub(crate) fn console_bytes(text: &str) -> Vec { + if cfg!(windows) { + text.replace('\n', "\r\n").into_bytes() + } else { + text.as_bytes().to_vec() + } +} + +/// Write the stderr then stdout text of a served invocation and flush both. +pub(crate) fn write_streams(stderr: &str, stdout: &str) -> io::Result<()> { + let mut error = io::stderr().lock(); + error.write_all(&console_bytes(stderr))?; + error.flush()?; + let mut output = io::stdout().lock(); + output.write_all(&console_bytes(stdout))?; + output.flush() +} + +/// Device and inode of standard output and error when they are regular files. +#[cfg(unix)] +pub(crate) fn redirected_output_files() -> Vec<(u64, u64)> { + use std::os::fd::AsFd; + use std::os::unix::fs::MetadataExt; + + let descriptors = [ + io::stdout().as_fd().try_clone_to_owned(), + io::stderr().as_fd().try_clone_to_owned(), + ]; + let mut identities: Vec<(u64, u64)> = Vec::new(); + for descriptor in descriptors.into_iter().flatten() { + if let Ok(metadata) = std::fs::File::from(descriptor).metadata() + && metadata.is_file() + { + identities.push((metadata.dev(), metadata.ino())); + } + } + identities.sort_unstable(); + identities.dedup(); + identities +} + +/// Output files are not excluded on this platform; a changed output file only misses reuse. +#[cfg(not(unix))] +pub(crate) fn redirected_output_files() -> Vec<(u64, u64)> { + Vec::new() +} diff --git a/crates/sqlbuild-cli/src/replay/_helpers/mod.rs b/crates/sqlbuild-cli/src/replay/_helpers/mod.rs new file mode 100644 index 0000000000..07f5c4f195 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod console; +pub(crate) mod records; +pub(crate) mod serving; diff --git a/crates/sqlbuild-cli/src/replay/_helpers/records.rs b/crates/sqlbuild-cli/src/replay/_helpers/records.rs new file mode 100644 index 0000000000..80d710eb69 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/_helpers/records.rs @@ -0,0 +1,95 @@ +//! Read launch records and turn their rules into the native reuse check's request. + +use std::path::{Path, PathBuf}; +use std::time::UNIX_EPOCH; + +use sqlbuild_cache::project_reuse::models::{HostIdentity, ReuseRules}; +use sqlbuild_cache::project_snapshot::models::SnapshotRules; + +use crate::replay::constants::LAUNCH_RECORD_FORMAT; +use crate::replay::models::{LaunchRecord, LaunchRules}; + +/// The record at `path` for `key`, or `None` when absent, unreadable or of another shape. +pub(crate) fn read_record(path: &Path, key: &str) -> Option { + let Ok(bytes) = std::fs::read(path) else { + return None; + }; + let Ok(record) = serde_json::from_slice::(&bytes) else { + return None; + }; + (record.format == LAUNCH_RECORD_FORMAT && record.key == key && startup_unchanged(&record)) + .then_some(record) +} + +fn startup_unchanged(record: &LaunchRecord) -> bool { + record + .startup_inputs + .iter() + .all(|(path, modified, size)| match std::fs::metadata(path) { + Ok(metadata) => { + let Ok(time) = metadata.modified() else { + return false; + }; + let Ok(duration) = time.duration_since(UNIX_EPOCH) else { + return false; + }; + let Ok(current) = i64::try_from(duration.as_nanos()) else { + return false; + }; + Some(current) == *modified && Some(metadata.len()) == *size + } + Err(error) => { + error.kind() == std::io::ErrorKind::NotFound && modified.is_none() && size.is_none() + } + }) +} + +/// The native reuse rules of a record, with this process's redirected output files. +pub(crate) fn reuse_rules(rules: &LaunchRules, output_files: Vec<(u64, u64)>) -> ReuseRules { + ReuseRules { + snapshot: SnapshotRules { + excluded: rules.excluded.clone(), + excluded_root: rules.excluded_root.clone(), + presence_suffixes: rules.presence_suffixes.clone(), + output_files, + }, + tracked_prefixes: rules.tracked_prefixes.clone(), + untracked_names: rules.untracked_names.clone(), + disabling_variables: rules.disabling_variables.clone(), + config_filenames: rules.config_filenames.clone(), + racy_window_ns: rules.racy_window_ns, + max_stored_entries: rules.max_stored_entries, + max_entry_bytes: rules.max_entry_bytes, + missing_environment_value: rules.missing_environment_value.clone(), + missing_file_digest: rules.missing_file_digest.clone(), + missing_path_mtime_ns: rules.missing_path_mtime_ns, + project_root_path_mtime_ns: rules.project_root_path_mtime_ns, + compiled_root: PathBuf::from(&rules.compiled_root), + retired_directories: rules.retired_directories.clone(), + } +} + +/// The host identity the Python process computed for this invocation. +pub(crate) fn host_identity(record: &LaunchRecord) -> HostIdentity { + HostIdentity { + invocation_digest: record.invocation_digest.clone(), + runtime: record.runtime.clone(), + search_path: record.search_path.clone(), + } +} + +/// `reuse_timings(outcome=HIT, check_ms)` plus `total_ms`, in Python's insertion order. +pub(crate) fn hit_timings( + record: &LaunchRecord, + check_ms: i64, + total_ms: i64, +) -> Vec<(String, i64)> { + let names = &record.timings; + vec![ + (names.hit.clone(), 1), + (names.miss.clone(), 0), + (names.bypass.clone(), 0), + (names.check.clone(), check_ms), + (names.total.clone(), total_ms), + ] +} diff --git a/crates/sqlbuild-cli/src/replay/_helpers/serving.rs b/crates/sqlbuild-cli/src/replay/_helpers/serving.rs new file mode 100644 index 0000000000..482f3b3893 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/_helpers/serving.rs @@ -0,0 +1,61 @@ +//! The native reuse check against a launch record and the output a hit replays. + +use std::path::PathBuf; +use std::time::Instant; + +use sqlbuild_cache::project_reuse::main::check_reuse::check_reuse; +use sqlbuild_cache::project_reuse::main::replay_stdout::replay_stdout; +use sqlbuild_cache::project_reuse::models::{ReuseCheck, ReuseOutcome, ReuseRules, StoredOutput}; + +use crate::replay::_helpers::console::tenths_of_seconds; +use crate::replay::_helpers::records::{hit_timings, host_identity}; +use crate::replay::models::{LaunchRecord, ServedInvocation}; + +fn elapsed_ms(started: Instant) -> i64 { + i64::try_from(started.elapsed().as_millis()).unwrap_or(i64::MAX) +} + +/// The stored output on a hit, or `None` when the slot was evicted or the check misses or bypasses. +pub(crate) fn stored_output(record: &LaunchRecord, rules: &ReuseRules) -> Option { + let store_path = PathBuf::from(&record.store_path); + if !store_path.is_file() { + return None; + } + let attempt = check_reuse(ReuseCheck { + project_dir: PathBuf::from(&record.project_dir), + store_path, + identity: host_identity(record), + bypass_requested: false, + json_output: record.json_output, + rules, + }); + if attempt.outcome != ReuseOutcome::Hit || attempt.replay_failed { + return None; + } + attempt.replay +} + +pub(crate) fn served( + record: &LaunchRecord, + output: &StoredOutput, + started: Instant, +) -> Option { + let check_ms = elapsed_ms(started); + let timings = if record.json_output { + hit_timings(record, check_ms, elapsed_ms(started)) + } else { + Vec::new() + }; + let stdout = replay_stdout(output, &timings)?; + let (prefix, suffix) = &record.hit_message; + let mut stderr = format!("{prefix}{}{suffix}\n", tenths_of_seconds(check_ms)); + for line in &output.stderr_lines { + stderr.push_str(line); + stderr.push('\n'); + } + Some(ServedInvocation { + stdout, + stderr, + exit_code: output.exit_code, + }) +} diff --git a/crates/sqlbuild-cli/src/replay/constants.rs b/crates/sqlbuild-cli/src/replay/constants.rs new file mode 100644 index 0000000000..2ee2e8da61 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/constants.rs @@ -0,0 +1,8 @@ +//! The launch record format shared with `sqlbuild.cli.entry.launch_records`. + +/// Bumped whenever the launch record changes shape; older records are ignored. +pub const LAUNCH_RECORD_FORMAT: i64 = 1; +/// Launch records live under `/target/cache//.json`. +pub const LAUNCH_RECORD_DIRECTORY: [&str; 3] = ["target", "cache", "launcher-v1"]; +pub const LAUNCH_RECORD_SUFFIX: &str = ".json"; +pub const OUTPUT_FAILURE_EXIT_CODE: i64 = 1; diff --git a/crates/sqlbuild-cli/src/replay/main/mod.rs b/crates/sqlbuild-cli/src/replay/main/mod.rs new file mode 100644 index 0000000000..6f3a35f1ad --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/main/mod.rs @@ -0,0 +1 @@ +pub mod serve_reused_compile; diff --git a/crates/sqlbuild-cli/src/replay/main/serve_reused_compile.rs b/crates/sqlbuild-cli/src/replay/main/serve_reused_compile.rs new file mode 100644 index 0000000000..868cd4bc2a --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/main/serve_reused_compile.rs @@ -0,0 +1,51 @@ +//! Replay the stored compile when the launch record still matches, as the Python CLI would. + +use std::io; +use std::path::{Path, PathBuf}; +use std::time::{Instant, SystemTime}; + +use crate::compute_logs::main::finish_capture::finish_capture; +use crate::compute_logs::main::start_capture::start_capture; +use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; +use crate::replay::_helpers::console::{redirected_output_files, write_streams}; +use crate::replay::_helpers::records::{read_record, reuse_rules}; +use crate::replay::_helpers::serving::{served, stored_output}; +use crate::replay::constants::OUTPUT_FAILURE_EXIT_CODE; + +/// The exit code of a natively served reuse hit, or `None` to hand the invocation to Python. +pub fn serve_reused_compile( + record_path: &Path, + key: &str, + started: (Instant, SystemTime), +) -> Option { + let record = read_record(record_path, key)?; + let rules = reuse_rules(&record.rules, redirected_output_files()); + let output = stored_output(&record, &rules)?; + let mut invocation = served(&record, &output, started.0)?; + let capture: Option> = record.capture.as_ref().map(|capture| { + start_capture(CaptureStart { + root: PathBuf::from(&capture.root), + command: capture.command.clone(), + project_dir: capture.project_dir.clone(), + target: capture.target.clone(), + started_at: started.1, + }) + }); + if let Err(error) = write_streams(&invocation.stderr, &invocation.stdout) { + let message = format!("Compile replay failed: could not write output: {error}\n"); + let _ = write_streams(&message, ""); + invocation.stderr.push_str(&message); + invocation.exit_code = OUTPUT_FAILURE_EXIT_CODE; + } + if let Some(Ok(capture)) = capture { + let _ = finish_capture( + capture, + &CaptureEnd { + stdout: invocation.stdout.into_bytes(), + stderr: invocation.stderr.into_bytes(), + exit_code: invocation.exit_code, + }, + ); + } + Some(invocation.exit_code) +} diff --git a/crates/sqlbuild-cli/src/replay/mod.rs b/crates/sqlbuild-cli/src/replay/mod.rs new file mode 100644 index 0000000000..97f4881e04 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/mod.rs @@ -0,0 +1,9 @@ +//! Serve a reused compile natively from the launch record the Python CLI left for this invocation. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/replay/models.rs b/crates/sqlbuild-cli/src/replay/models.rs new file mode 100644 index 0000000000..c2b4f0a5b0 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/models.rs @@ -0,0 +1,72 @@ +//! The launch record the Python CLI writes after a reusable compile. + +use std::collections::BTreeMap; + +use serde::Deserialize; + +/// The reuse constants the Python host owns, as `reuse_rules()` returns them. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +pub struct LaunchRules { + pub excluded: Vec, + pub excluded_root: Vec, + pub presence_suffixes: Vec, + pub tracked_prefixes: Vec, + pub untracked_names: Vec, + pub disabling_variables: Vec<(String, String)>, + pub config_filenames: Vec, + pub racy_window_ns: i64, + pub max_stored_entries: usize, + pub max_entry_bytes: usize, + pub missing_environment_value: String, + pub missing_file_digest: String, + pub missing_path_mtime_ns: i64, + pub project_root_path_mtime_ns: i64, + pub compiled_root: String, + pub retired_directories: Vec, +} + +/// Where the Python CLI captured this invocation's compute logs. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +pub struct LaunchCapture { + pub root: String, + pub command: String, + pub project_dir: String, + pub target: Option, +} + +/// The `compile_timings` names a JSON replay reports. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +pub struct LaunchTimings { + pub hit: String, + pub miss: String, + pub bypass: String, + pub check: String, + pub total: String, +} + +/// Everything a reuse hit needs that only the Python process could know. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +pub struct LaunchRecord { + pub format: i64, + pub key: String, + pub project_dir: String, + pub store_path: String, + pub invocation_digest: String, + pub runtime: BTreeMap, + pub search_path: Vec, + pub startup_inputs: Vec<(String, Option, Option)>, + pub json_output: bool, + pub rules: LaunchRules, + pub capture: Option, + /// The hit notice before and after its `{seconds:.1f}` field. + pub hit_message: (String, String), + pub timings: LaunchTimings, +} + +/// One served invocation: what was written and the exit code. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ServedInvocation { + pub stdout: String, + pub stderr: String, + pub exit_code: i64, +} diff --git a/crates/sqlbuild-cli/src/replay/tests.rs b/crates/sqlbuild-cli/src/replay/tests.rs new file mode 100644 index 0000000000..46c3c1617c --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_console.rs"] +mod console; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/replay/tests/test_console.rs b/crates/sqlbuild-cli/src/replay/tests/test_console.rs new file mode 100644 index 0000000000..c8cd5eb45f --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/tests/test_console.rs @@ -0,0 +1,37 @@ +use crate::replay::_helpers::console::tenths_of_seconds; +use crate::replay::tests::test_types::TenthsTestCase; + +#[test] +fn given_elapsed_milliseconds_when_formatting_seconds_then_matches_python_rounding() { + let test_cases = [ + TenthsTestCase { + description: "exact binary tie rounds to even", + milliseconds: 250, + expected_text: "0.2", + }, + TenthsTestCase { + description: "inexact half rounds on the binary value", + milliseconds: 350, + expected_text: "0.3", + }, + TenthsTestCase { + description: "upper exact tie rounds to even", + milliseconds: 750, + expected_text: "0.8", + }, + TenthsTestCase { + description: "seconds above one", + milliseconds: 2_450, + expected_text: "2.5", + }, + ]; + + for test_case in test_cases { + assert_eq!( + tenths_of_seconds(test_case.milliseconds), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/replay/tests/test_types.rs b/crates/sqlbuild-cli/src/replay/tests/test_types.rs new file mode 100644 index 0000000000..b8145c3236 --- /dev/null +++ b/crates/sqlbuild-cli/src/replay/tests/test_types.rs @@ -0,0 +1,6 @@ +/// Elapsed milliseconds and Python's `f"{ms / 1000:.1f}"`. +pub(crate) struct TenthsTestCase { + pub(crate) description: &'static str, + pub(crate) milliseconds: i64, + pub(crate) expected_text: &'static str, +} diff --git a/crates/sqlbuild-config/Cargo.toml b/crates/sqlbuild-config/Cargo.toml index 3b6b97cb00..b8a3ee52b9 100644 --- a/crates/sqlbuild-config/Cargo.toml +++ b/crates/sqlbuild-config/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true saphyr-parser.workspace = true toml_edit.workspace = true toml_parser.workspace = true +sqlbuild-core.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-config/src/constants.rs b/crates/sqlbuild-config/src/constants.rs index 11688decd0..62c7804891 100644 --- a/crates/sqlbuild-config/src/constants.rs +++ b/crates/sqlbuild-config/src/constants.rs @@ -9,3 +9,4 @@ pub(crate) const MAX_MINUTE: i64 = 59; pub(crate) const MAX_SECOND: i64 = 59; pub(crate) const MAX_MICROSECOND: i64 = 999_999; pub(crate) const BYTE_ORDER_MARK: char = '\u{feff}'; +pub(crate) const DECIMAL_BASE: u32 = 10; diff --git a/crates/sqlbuild-config/src/effective/_helpers/entries.rs b/crates/sqlbuild-config/src/effective/_helpers/entries.rs new file mode 100644 index 0000000000..e473631c3e --- /dev/null +++ b/crates/sqlbuild-config/src/effective/_helpers/entries.rs @@ -0,0 +1,37 @@ +//! Python `dict.update` over ordered entries. + +use std::collections::HashMap; +use std::hash::Hash; + +use crate::effective::types::Entries; + +/// `into.update(from)`: existing keys keep their position and take the new value. +pub(crate) fn updated( + into: Entries, + from: &[(K, V)], +) -> Entries { + let mut entries: Entries = into; + let mut positions: HashMap = entries + .iter() + .enumerate() + .map(|(position, (key, _))| (key.clone(), position)) + .collect(); + for (key, value) in from { + match positions.get(key) { + Some(position) => entries[*position].1 = value.clone(), + None => { + positions.insert(key.clone(), entries.len()); + entries.push((key.clone(), value.clone())); + } + } + } + entries +} + +/// `mapping.get(key)`. +pub(crate) fn lookup<'a, K: Eq, V>(entries: &'a [(K, V)], key: &K) -> Option<&'a V> { + entries + .iter() + .find(|(candidate, _)| candidate == key) + .map(|(_, value)| value) +} diff --git a/crates/sqlbuild-config/src/effective/_helpers/mod.rs b/crates/sqlbuild-config/src/effective/_helpers/mod.rs new file mode 100644 index 0000000000..269c2824a5 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod entries; diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs new file mode 100644 index 0000000000..6767150114 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -0,0 +1,185 @@ +//! Every `TargetConfig` field and how a local target overrides it. + +use crate::effective::models::{MergeRule, TargetField}; + +const RETENTION: &[&str] = &[ + "time_travel_retention", + "time_travel_retention_by_materialization", +]; +const LIMITS: Option<&str> = Some("execution_limits"); +const CLONE: Option<&str> = Some("clone"); + +/// `TargetConfig`'s fields in declaration order, nested dataclass fields under their group. +pub(crate) const TARGET_FIELDS: &[TargetField] = &[ + TargetField { + group: None, + name: "connection", + rule: MergeRule::Union, + }, + TargetField { + group: None, + name: "connection_name", + rule: MergeRule::Overlay { + decided_by: &["connection_name"], + }, + }, + TargetField { + group: None, + name: "vars", + rule: MergeRule::Union, + }, + TargetField { + group: None, + name: "database", + rule: MergeRule::Overlay { + decided_by: &["database"], + }, + }, + TargetField { + group: None, + name: "schema", + rule: MergeRule::Overlay { + decided_by: &["schema"], + }, + }, + TargetField { + group: None, + name: "loader_schema", + rule: MergeRule::Overlay { + decided_by: &["loader_schema"], + }, + }, + TargetField { + group: None, + name: "defer_sources_to", + rule: MergeRule::Overlay { + decided_by: &["defer_sources_to"], + }, + }, + TargetField { + group: None, + name: "defer_clone_from", + rule: MergeRule::Overlay { + decided_by: &["defer_clone_from"], + }, + }, + TargetField { + group: CLONE, + name: "allow_as_clone_origin", + rule: MergeRule::Overlay { + decided_by: &["allow_as_clone_origin"], + }, + }, + TargetField { + group: CLONE, + name: "allow_as_clone_destination", + rule: MergeRule::Overlay { + decided_by: &["allow_as_clone_destination"], + }, + }, + TargetField { + group: None, + name: "compile_cache", + rule: MergeRule::Overlay { + decided_by: &["compile_cache"], + }, + }, + TargetField { + group: None, + name: "time_travel_retention", + rule: MergeRule::Overlay { + decided_by: RETENTION, + }, + }, + TargetField { + group: None, + name: "time_travel_retention_by_materialization", + rule: MergeRule::ReplaceOrEmpty { + decided_by: RETENTION, + }, + }, + TargetField { + group: None, + name: "owns_time_travel_retention_namespace", + rule: MergeRule::Overlay { + decided_by: &["owns_time_travel_retention_namespace"], + }, + }, + TargetField { + group: None, + name: "default_table_type", + rule: MergeRule::Overlay { + decided_by: &["default_table_type"], + }, + }, + TargetField { + group: None, + name: "table_type_downgrade", + rule: MergeRule::Overlay { + decided_by: &["table_type_downgrade"], + }, + }, + TargetField { + group: None, + name: "time_travel_retention_decrease", + rule: MergeRule::Overlay { + decided_by: &["time_travel_retention_decrease"], + }, + }, + TargetField { + group: None, + name: "missing_migration_origin", + rule: MergeRule::Overlay { + decided_by: &["missing_migration_origin"], + }, + }, + TargetField { + group: LIMITS, + name: "max_models", + rule: MergeRule::Overlay { + decided_by: &["max_models"], + }, + }, + TargetField { + group: LIMITS, + name: "max_duration", + rule: MergeRule::Overlay { + decided_by: &["max_duration"], + }, + }, + TargetField { + group: LIMITS, + name: "max_duration_seconds", + rule: MergeRule::Overlay { + decided_by: &["max_duration"], + }, + }, + TargetField { + group: LIMITS, + name: "remediation", + rule: MergeRule::Overlay { + decided_by: &["remediation"], + }, + }, + TargetField { + group: Some("diff"), + name: "max_full_rows", + rule: MergeRule::Overlay { + decided_by: &["max_full_rows"], + }, + }, + TargetField { + group: Some("warehouses"), + name: "build", + rule: MergeRule::Overlay { + decided_by: &["build"], + }, + }, + TargetField { + group: Some("warehouses"), + name: "query", + rule: MergeRule::Overlay { + decided_by: &["query"], + }, + }, +]; diff --git a/crates/sqlbuild-config/src/effective/errors.rs b/crates/sqlbuild-config/src/effective/errors.rs new file mode 100644 index 0000000000..4dfe1c0e93 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/errors.rs @@ -0,0 +1,68 @@ +//! Why an effective configuration cannot be resolved. + +/// A precedence failure, carrying the authored name it is about. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EffectiveConfigError { + /// The selected target is defined in neither config file. + UnknownTarget(K), + /// A target selects a connection defined in neither config file. + UnknownConnection(K), + /// A target selects no connection while several named connections exist. + AmbiguousConnection { target: K, names: Vec }, +} + +/// One piece of a failure message: fixed text or an authored name, kept exact. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum MessagePart<'a, K> { + Text(&'static str), + Name(&'a K), +} + +impl EffectiveConfigError { + /// The pieces of the message Python's `SpecConfigError` or `CompileInputError` carries. + pub fn message_parts(&self) -> Vec> { + use MessagePart::{Name, Text}; + match self { + Self::UnknownTarget(name) => vec![Text("Unknown target '"), Name(name), Text("'")], + Self::UnknownConnection(name) => vec![ + Text("Unknown connection '"), + Name(name), + Text("' selected by target. Define [connections."), + Name(name), + Text("] in sqlbuild_project.toml or sqlbuild_local.toml."), + ], + Self::AmbiguousConnection { target, names } => { + let mut parts = vec![ + Text("targets."), + Name(target), + Text(" does not set connection and several named connections are defined ("), + ]; + for (position, name) in names.iter().enumerate() { + if position > 0 { + parts.push(Text(", ")); + } + parts.push(Name(name)); + } + parts.extend([ + Text("); set connection = \"\" on [targets."), + Name(target), + Text("] in sqlbuild_project.toml or sqlbuild_local.toml"), + ]); + parts + } + } + } +} + +impl> EffectiveConfigError { + /// The message text, for names that are valid text. + pub fn message(&self) -> String { + self.message_parts() + .into_iter() + .map(|part| match part { + MessagePart::Text(text) => text, + MessagePart::Name(name) => name.as_ref(), + }) + .collect() + } +} diff --git a/crates/sqlbuild-config/src/effective/main/default_connection_name.rs b/crates/sqlbuild-config/src/effective/main/default_connection_name.rs new file mode 100644 index 0000000000..fbf6b693a6 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/default_connection_name.rs @@ -0,0 +1,29 @@ +//! `_apply_default_connection`: a target without a connection uses the only named one. + +use std::collections::BTreeSet; + +use crate::effective::errors::EffectiveConfigError; +use crate::effective::models::ConnectionChoice; + +/// The only named connection a target defaults to, `None` when it keeps its own, or ambiguity. +pub fn default_connection_name( + choice: ConnectionChoice, +) -> Result, EffectiveConfigError> { + if choice.selected { + return Ok(None); + } + let names: BTreeSet = choice + .project_names + .into_iter() + .chain(choice.local_names) + .collect(); + let mut names: Vec = names.into_iter().collect(); + match names.len() { + 0 => Ok(None), + 1 => Ok(names.pop()), + _ => Err(EffectiveConfigError::AmbiguousConnection { + target: choice.target, + names, + }), + } +} diff --git a/crates/sqlbuild-config/src/effective/main/effective_connection.rs b/crates/sqlbuild-config/src/effective/main/effective_connection.rs new file mode 100644 index 0000000000..38ab63062c --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/effective_connection.rs @@ -0,0 +1,28 @@ +//! `build_effective_connection`: the project connection with every override, before expansion. + +use std::hash::Hash; + +use crate::effective::_helpers::entries::{lookup, updated}; +use crate::effective::errors::EffectiveConfigError; +use crate::effective::models::ConnectionLayers; +use crate::effective::types::Entries; + +/// The merged connection, or the unknown connection a target selects. +pub fn effective_connection( + layers: ConnectionLayers, +) -> Result, EffectiveConfigError> { + let mut connection: Entries = layers.project; + if let Some((name, target)) = layers.target { + if let Some(name) = name { + let project = lookup(&layers.project_named, &name); + let local = lookup(&layers.local_named, &name); + if project.is_none() && local.is_none() { + return Err(EffectiveConfigError::UnknownConnection(name)); + } + connection = updated(connection, project.map(Vec::as_slice).unwrap_or_default()); + connection = updated(connection, local.map(Vec::as_slice).unwrap_or_default()); + } + connection = updated(connection, &target); + } + Ok(updated(connection, &layers.local)) +} diff --git a/crates/sqlbuild-config/src/effective/main/effective_settings.rs b/crates/sqlbuild-config/src/effective/main/effective_settings.rs new file mode 100644 index 0000000000..e05d49231f --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/effective_settings.rs @@ -0,0 +1,25 @@ +//! `build_effective_settings`: project settings with the local settings a file overrides. + +use std::hash::Hash; + +use crate::effective::_helpers::entries::{lookup, updated}; +use crate::effective::types::Entries; + +/// The local value of one overridden setting. +fn local_setting(local: &[(K, V)], name: &K) -> Option<(K, V)> { + lookup(local, name).map(|value| (name.clone(), value.clone())) +} + +/// The project settings, each overridden name taking the local value. +#[must_use] +pub fn effective_settings( + project: Entries, + local: &[(K, V)], + overrides: &[K], +) -> Entries { + let overridden: Entries = overrides + .iter() + .filter_map(|name| local_setting(local, name)) + .collect(); + updated(project, &overridden) +} diff --git a/crates/sqlbuild-config/src/effective/main/effective_target_name.rs b/crates/sqlbuild-config/src/effective/main/effective_target_name.rs new file mode 100644 index 0000000000..e7d0d8d2fc --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/effective_target_name.rs @@ -0,0 +1,18 @@ +//! `resolve_target_name`: the selected, local, then project default target. + +use crate::effective::errors::EffectiveConfigError; +use crate::effective::models::TargetChoice; + +/// The effective target name, `None` when nothing selects one, or why the name is unknown. +pub fn effective_target_name( + choice: TargetChoice, +) -> Result, EffectiveConfigError> { + let Some(name) = choice.selected.or(choice.local).or(choice.default) else { + return Ok(None); + }; + if choice.defined.contains(&name) { + Ok(Some(name)) + } else { + Err(EffectiveConfigError::UnknownTarget(name)) + } +} diff --git a/crates/sqlbuild-config/src/effective/main/effective_vars.rs b/crates/sqlbuild-config/src/effective/main/effective_vars.rs new file mode 100644 index 0000000000..e0b25d9d27 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/effective_vars.rs @@ -0,0 +1,14 @@ +//! `build_effective_vars`: project, target, local, then CLI vars, before template expansion. + +use std::hash::Hash; + +use crate::effective::_helpers::entries::updated; +use crate::effective::types::Entries; + +/// The vars of every layer merged, lowest precedence first, with `dict.update` semantics. +#[must_use] +pub fn effective_vars(layers: &[&[(K, V)]]) -> Entries { + layers + .iter() + .fold(Vec::new(), |vars, layer| updated(vars, layer)) +} diff --git a/crates/sqlbuild-config/src/effective/main/merged_target.rs b/crates/sqlbuild-config/src/effective/main/merged_target.rs new file mode 100644 index 0000000000..b9c1a17b7a --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/merged_target.rs @@ -0,0 +1,64 @@ +//! `_merge_target_config`: a project target with its local overrides. + +use std::hash::Hash; + +use crate::effective::_helpers::entries::updated; +use crate::effective::constants::TARGET_FIELDS; +use crate::effective::models::{FieldValue, MergeRule, TargetField, TargetLayer}; + +/// Whether the local layer sets the field of `group` named `name`. +fn sets(local: &TargetLayer, group: Option<&str>, name: &str) -> bool { + TARGET_FIELDS + .iter() + .zip(&local.fields) + .find(|(field, _)| field.group == group && field.name == name) + .is_some_and(|(_, value)| value.is_set()) +} + +/// One field's merged value. +fn merged_field( + field: &TargetField, + project: &FieldValue, + local: &FieldValue, + local_layer: &TargetLayer, +) -> FieldValue { + let decides = |decided_by: &[&str]| { + decided_by + .iter() + .any(|name| sets(local_layer, field.group, name)) + }; + match (field.rule, project, local) { + (MergeRule::Union, FieldValue::Map(project), FieldValue::Map(local)) => { + let merged = updated( + project.clone().unwrap_or_default(), + local.as_deref().unwrap_or_default(), + ); + FieldValue::Map(Some(merged)) + } + (MergeRule::ReplaceOrEmpty { decided_by }, _, FieldValue::Map(local)) + if decides(decided_by) => + { + FieldValue::Map(Some(local.clone().unwrap_or_default())) + } + (MergeRule::Overlay { decided_by }, _, _) if decides(decided_by) => local.clone(), + _ => project.clone(), + } +} + +/// The project target, or with a local target each field merged by its rule. +#[must_use] +pub fn merged_target( + project: TargetLayer, + local: Option<&TargetLayer>, +) -> TargetLayer { + let Some(local) = local else { + return project; + }; + TargetLayer { + fields: TARGET_FIELDS + .iter() + .zip(project.fields.iter().zip(&local.fields)) + .map(|(field, (project, local_value))| merged_field(field, project, local_value, local)) + .collect(), + } +} diff --git a/crates/sqlbuild-config/src/effective/main/mod.rs b/crates/sqlbuild-config/src/effective/main/mod.rs new file mode 100644 index 0000000000..80f728929d --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/mod.rs @@ -0,0 +1,7 @@ +pub mod default_connection_name; +pub mod effective_connection; +pub mod effective_settings; +pub mod effective_target_name; +pub mod effective_vars; +pub mod merged_target; +pub mod target_fields; diff --git a/crates/sqlbuild-config/src/effective/main/target_fields.rs b/crates/sqlbuild-config/src/effective/main/target_fields.rs new file mode 100644 index 0000000000..3b19743d47 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/target_fields.rs @@ -0,0 +1,10 @@ +//! The `TargetConfig` fields a target layer holds, in order. + +use crate::effective::constants::TARGET_FIELDS; +use crate::effective::models::TargetField; + +/// Every field of `TargetConfig`, nested dataclass fields under their group. +#[must_use] +pub fn target_fields() -> &'static [TargetField] { + TARGET_FIELDS +} diff --git a/crates/sqlbuild-config/src/effective/mod.rs b/crates/sqlbuild-config/src/effective/mod.rs new file mode 100644 index 0000000000..c8eea8870d --- /dev/null +++ b/crates/sqlbuild-config/src/effective/mod.rs @@ -0,0 +1,11 @@ +//! Effective configuration precedence: target selection, target merge, vars, connection, settings. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/effective/models.rs b/crates/sqlbuild-config/src/effective/models.rs new file mode 100644 index 0000000000..34cb57057d --- /dev/null +++ b/crates/sqlbuild-config/src/effective/models.rs @@ -0,0 +1,82 @@ +//! Configuration layers, with authored values kept as opaque handles of type `V`. + +use crate::effective::types::Entries; + +/// How a target field combines the project layer with the local layer. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MergeRule { + /// `{**project, **local}`. + Union, + /// The local value when any `decided_by` local field is set, else the project value. + Overlay { decided_by: &'static [&'static str] }, + /// Like `Overlay`, but an unset local mapping becomes empty instead of falling back. + ReplaceOrEmpty { decided_by: &'static [&'static str] }, +} + +impl MergeRule { + /// Whether the field holds a mapping rather than a scalar. + pub fn is_map(self) -> bool { + !matches!(self, Self::Overlay { .. }) + } +} + +/// One `TargetConfig` field: its nested dataclass, if any, its name and its merge rule. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TargetField { + pub group: Option<&'static str>, + pub name: &'static str, + pub rule: MergeRule, +} + +/// One target field's authored value; `None` is a local layer leaving it unset. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FieldValue { + Scalar(Option), + Map(Option>), +} + +impl FieldValue { + /// Whether the layer sets this field. + pub fn is_set(&self) -> bool { + match self { + Self::Scalar(value) => value.is_some(), + Self::Map(value) => value.is_some(), + } + } +} + +/// One layer of a target: each field of `TARGET_FIELDS`, in order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TargetLayer { + pub fields: Vec>, +} + +/// Every connection layer, lowest precedence first. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConnectionLayers { + pub project: Entries, + /// The selected target's connection name and connection, when a target is selected. + pub target: Option<(Option, Entries)>, + pub project_named: Entries>, + pub local_named: Entries>, + pub local: Entries, +} + +/// Where a target's connection comes from, for the only-named-connection default. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConnectionChoice { + pub target: K, + /// Whether the target, project or local config already selects or authors a connection. + pub selected: bool, + pub project_names: Vec, + pub local_names: Vec, +} + +/// The names that choose a target, highest precedence first, and the targets defined. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TargetChoice { + pub selected: Option, + pub local: Option, + pub default: Option, + pub defined: Vec, +} diff --git a/crates/sqlbuild-config/src/effective/tests.rs b/crates/sqlbuild-config/src/effective/tests.rs new file mode 100644 index 0000000000..80aecb6202 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_effective.rs"] +mod effective; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/effective/tests/helpers.rs b/crates/sqlbuild-config/src/effective/tests/helpers.rs new file mode 100644 index 0000000000..b4be2fe171 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/tests/helpers.rs @@ -0,0 +1,63 @@ +use crate::effective::main::target_fields::target_fields; +use crate::effective::models::{FieldValue, MergeRule, TargetLayer}; +use crate::effective::tests::test_types::Pairs; +use crate::effective::types::Entries; + +pub(super) fn entries(pairs: Pairs) -> Entries<&'static str, i64> { + pairs.to_vec() +} + +/// A project layer: every scalar set to its field position, every map holding one entry. +pub(super) fn project_layer() -> TargetLayer<&'static str, i64> { + TargetLayer { + fields: target_fields() + .iter() + .enumerate() + .map(|(position, field)| { + let value: i64 = i64::try_from(position).expect("few fields"); + layer_value(field.rule, Some(value), "project") + }) + .collect(), + } +} + +/// The field value of a rule's shape: a scalar, or a one-entry map under `key`. +fn layer_value( + rule: MergeRule, + value: Option, + key: &'static str, +) -> FieldValue<&'static str, i64> { + let shapes: [FieldValue<&'static str, i64>; 2] = [ + FieldValue::Scalar(value), + FieldValue::Map(value.map(|value| vec![(key, value)])), + ]; + shapes[usize::from(rule.is_map())].clone() +} + +/// A local layer leaving every field unset except `set`; maps hold `("local", value)`. +pub(super) fn local_layer(set: &[(&str, i64)]) -> TargetLayer<&'static str, i64> { + TargetLayer { + fields: target_fields() + .iter() + .map(|field| { + let value: Option = set + .iter() + .find(|(name, _)| *name == field.name) + .map(|(_, value)| *value); + layer_value(field.rule, value, "local") + }) + .collect(), + } +} + +/// The merged value of field `name`. +pub(super) fn field_value( + layer: &TargetLayer<&'static str, i64>, + name: &str, +) -> FieldValue<&'static str, i64> { + let position: usize = target_fields() + .iter() + .position(|field| field.name == name) + .expect("known field"); + layer.fields[position].clone() +} diff --git a/crates/sqlbuild-config/src/effective/tests/test_effective.rs b/crates/sqlbuild-config/src/effective/tests/test_effective.rs new file mode 100644 index 0000000000..e4c38903f3 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/tests/test_effective.rs @@ -0,0 +1,297 @@ +use crate::effective::errors::EffectiveConfigError; +use crate::effective::main::default_connection_name::default_connection_name; +use crate::effective::main::effective_connection::effective_connection; +use crate::effective::main::effective_settings::effective_settings; +use crate::effective::main::effective_target_name::effective_target_name; +use crate::effective::main::effective_vars::effective_vars; +use crate::effective::main::merged_target::merged_target; +use crate::effective::models::{ + ConnectionChoice, ConnectionLayers, FieldValue, TargetChoice, TargetLayer, +}; +use crate::effective::tests::helpers::{entries, field_value, local_layer, project_layer}; +use crate::effective::tests::test_types::{ + ConnectionTestCase, DefaultConnectionTestCase, MergeTestCase, MessageTestCase, Pairs, + TargetNameTestCase, UnionTestCase, +}; +use crate::effective::types::Entries; + +#[test] +fn given_target_names_when_choosing_then_cli_beats_local_beats_project_default() { + let test_cases = [ + TargetNameTestCase { + description: "the CLI selection wins", + selected: Some("ci"), + local: Some("dev"), + default: Some("prod"), + expected_result: Ok(Some("ci")), + }, + TargetNameTestCase { + description: "the local target beats the project default", + selected: None, + local: Some("dev"), + default: Some("prod"), + expected_result: Ok(Some("dev")), + }, + TargetNameTestCase { + description: "nothing selects a target", + selected: None, + local: None, + default: None, + expected_result: Ok(None), + }, + TargetNameTestCase { + description: "an undefined name is rejected", + selected: Some("qa"), + local: None, + default: None, + expected_result: Err(EffectiveConfigError::UnknownTarget("qa")), + }, + ]; + for test_case in test_cases { + let choice: TargetChoice<&str> = TargetChoice { + selected: test_case.selected, + local: test_case.local, + default: test_case.default, + defined: vec!["ci", "dev", "prod"], + }; + + assert_eq!( + effective_target_name(choice), + test_case.expected_result, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_connection_layers_when_merging_then_later_layers_override_in_place() { + let test_cases = [ + ConnectionTestCase { + description: "no target: project then local connection", + target_name: None, + project_named: &[], + local_named: &[], + expected_result: Ok(&[("host", 9), ("port", 1), ("user", 9)]), + }, + ConnectionTestCase { + description: "a named connection applies project then local entries before the target", + target_name: Some("warehouse"), + project_named: &[("warehouse", &[("port", 2), ("role", 2)])], + local_named: &[("warehouse", &[("role", 3)])], + expected_result: Ok(&[ + ("host", 9), + ("port", 2), + ("role", 3), + ("db", 4), + ("user", 9), + ]), + }, + ConnectionTestCase { + description: "an undefined named connection is rejected", + target_name: Some("missing"), + project_named: &[("warehouse", &[])], + local_named: &[], + expected_result: Err(EffectiveConfigError::UnknownConnection("missing")), + }, + ]; + for test_case in test_cases { + let named = |pairs: &'static [(&'static str, Pairs)]| -> Entries<&str, Entries<&str, i64>> { + pairs + .iter() + .map(|(name, pairs)| (*name, entries(pairs))) + .collect() + }; + let layers: ConnectionLayers<&str, i64> = ConnectionLayers { + project: entries(&[("host", 1), ("port", 1)]), + target: test_case + .target_name + .map(|name| (Some(name), entries(&[("db", 4)]))), + project_named: named(test_case.project_named), + local_named: named(test_case.local_named), + local: entries(&[("host", 9), ("user", 9)]), + }; + + assert_eq!( + effective_connection(layers), + test_case.expected_result.map(entries), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_named_connections_when_target_selects_none_then_only_one_is_the_default() { + let test_cases = [ + DefaultConnectionTestCase { + description: "a target that selects a connection keeps it", + selected: true, + project_names: &["a", "b"], + local_names: &[], + expected_result: Ok(None), + }, + DefaultConnectionTestCase { + description: "the same name in both files is one connection", + selected: false, + project_names: &["a"], + local_names: &["a"], + expected_result: Ok(Some("a")), + }, + DefaultConnectionTestCase { + description: "several names are ambiguous and listed in sorted order", + selected: false, + project_names: &["b"], + local_names: &["a"], + expected_result: Err(EffectiveConfigError::AmbiguousConnection { + target: "dev", + names: vec!["a", "b"], + }), + }, + ]; + for test_case in test_cases { + let choice: ConnectionChoice<&str> = ConnectionChoice { + target: "dev", + selected: test_case.selected, + project_names: test_case.project_names.to_vec(), + local_names: test_case.local_names.to_vec(), + }; + + assert_eq!( + default_connection_name(choice), + test_case.expected_result, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_local_target_when_merging_then_set_fields_override_by_their_rule() { + let test_cases = [ + MergeTestCase { + description: "an empty local target keeps project scalars and maps", + local: &[], + expected_result: &[ + ("database", || FieldValue::Scalar(Some(3))), + ("connection", || FieldValue::Map(Some(vec![("project", 0)]))), + ("max_duration_seconds", || FieldValue::Scalar(Some(20))), + ], + }, + MergeTestCase { + description: "a local scalar overrides and maps union with local entries last", + local: &[("database", 30), ("vars", 7)], + expected_result: &[ + ("database", || FieldValue::Scalar(Some(30))), + ("vars", || { + FieldValue::Map(Some(vec![("project", 2), ("local", 7)])) + }), + ("schema", || FieldValue::Scalar(Some(4))), + ], + }, + MergeTestCase { + description: "a local max duration decides its seconds even when unset", + local: &[("max_duration", 5)], + expected_result: &[ + ("max_duration", || FieldValue::Scalar(Some(5))), + ("max_duration_seconds", || FieldValue::Scalar(None)), + ], + }, + MergeTestCase { + description: "a local retention empties the unset per-materialization retention", + local: &[("time_travel_retention", 8)], + expected_result: &[ + ("time_travel_retention", || FieldValue::Scalar(Some(8))), + ("time_travel_retention_by_materialization", || { + FieldValue::Map(Some(Vec::new())) + }), + ], + }, + MergeTestCase { + description: "a local per-materialization retention clears the project retention", + local: &[("time_travel_retention_by_materialization", 6)], + expected_result: &[ + ("time_travel_retention", || FieldValue::Scalar(None)), + ("time_travel_retention_by_materialization", || { + FieldValue::Map(Some(vec![("local", 6)])) + }), + ], + }, + ]; + for test_case in test_cases { + let local: TargetLayer<&str, i64> = local_layer(test_case.local); + + let merged: TargetLayer<&str, i64> = merged_target(project_layer(), Some(&local)); + + let values: Vec<(&str, FieldValue<&str, i64>)> = test_case + .expected_result + .iter() + .map(|(name, _)| (*name, field_value(&merged, name))) + .collect(); + let expected: Vec<(&str, FieldValue<&str, i64>)> = test_case + .expected_result + .iter() + .map(|(name, value)| (*name, value())) + .collect(); + assert_eq!( + values, + expected, + "{} {:?}", + test_case.description, + test_case.expected_result.len() + ); + } +} + +#[test] +fn given_layers_when_merging_vars_and_settings_then_dict_update_order_holds() { + let test_cases = [UnionTestCase { + description: "keys keep first position and take the last value", + layers: &[&[("a", 1), ("b", 1)], &[("c", 2)], &[("a", 3)]], + expected_result: &[("a", 3), ("b", 1), ("c", 2)], + }]; + for test_case in test_cases { + let layers: Vec<&[(&str, i64)]> = test_case.layers.to_vec(); + + assert_eq!( + effective_vars(&layers), + entries(test_case.expected_result), + "{}", + test_case.description + ); + assert_eq!( + effective_settings( + entries(&[("a", 1), ("b", 1)]), + &[("a", 5), ("b", 5)], + &["b"] + ), + entries(&[("a", 1), ("b", 5)]), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_ambiguous_connection_when_rendering_then_names_join_into_the_python_message() { + let test_cases = [MessageTestCase { + description: "names are inserted unchanged between the fixed text", + names: &["a", "b"], + expected_message: "targets.dev does not set connection and several named connections are \ + defined (a, b); set connection = \"\" on [targets.dev] in \ + sqlbuild_project.toml or sqlbuild_local.toml", + }]; + for test_case in test_cases { + let error: EffectiveConfigError<&str> = EffectiveConfigError::AmbiguousConnection { + target: "dev", + names: test_case.names.to_vec(), + }; + + assert_eq!( + error.message(), + test_case.expected_message, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/effective/tests/test_types.rs b/crates/sqlbuild-config/src/effective/tests/test_types.rs new file mode 100644 index 0000000000..0199cdea3a --- /dev/null +++ b/crates/sqlbuild-config/src/effective/tests/test_types.rs @@ -0,0 +1,50 @@ +use crate::effective::errors::EffectiveConfigError; +use crate::effective::models::FieldValue; + +pub(super) type Pairs = &'static [(&'static str, i64)]; +/// Builds one expected merged field value. +pub(super) type ExpectedField = fn() -> FieldValue<&'static str, i64>; + +pub(super) struct TargetNameTestCase { + pub(super) description: &'static str, + pub(super) selected: Option<&'static str>, + pub(super) local: Option<&'static str>, + pub(super) default: Option<&'static str>, + pub(super) expected_result: Result, EffectiveConfigError<&'static str>>, +} + +pub(super) struct ConnectionTestCase { + pub(super) description: &'static str, + pub(super) target_name: Option<&'static str>, + pub(super) project_named: &'static [(&'static str, Pairs)], + pub(super) local_named: &'static [(&'static str, Pairs)], + pub(super) expected_result: Result>, +} + +pub(super) struct DefaultConnectionTestCase { + pub(super) description: &'static str, + pub(super) selected: bool, + pub(super) project_names: &'static [&'static str], + pub(super) local_names: &'static [&'static str], + pub(super) expected_result: Result, EffectiveConfigError<&'static str>>, +} + +pub(super) struct MergeTestCase { + pub(super) description: &'static str, + /// Local fields to set, by name, with their values. + pub(super) local: &'static [(&'static str, i64)], + /// The merged value of each named field. + pub(super) expected_result: &'static [(&'static str, ExpectedField)], +} + +pub(super) struct UnionTestCase { + pub(super) description: &'static str, + pub(super) layers: &'static [Pairs], + pub(super) expected_result: Pairs, +} + +pub(super) struct MessageTestCase { + pub(super) description: &'static str, + pub(super) names: &'static [&'static str], + pub(super) expected_message: &'static str, +} diff --git a/crates/sqlbuild-config/src/effective/types.rs b/crates/sqlbuild-config/src/effective/types.rs new file mode 100644 index 0000000000..7ff1409e89 --- /dev/null +++ b/crates/sqlbuild-config/src/effective/types.rs @@ -0,0 +1,4 @@ +//! Ordered configuration mapping aliases. + +/// `str`-keyed entries in insertion order, read and written with Python `dict` semantics. +pub type Entries = Vec<(K, V)>; diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs index bdbf4ddbf3..cb4d5a46a8 100644 --- a/crates/sqlbuild-config/src/lib.rs +++ b/crates/sqlbuild-config/src/lib.rs @@ -3,6 +3,7 @@ #![forbid(unsafe_code)] pub(crate) mod constants; +pub mod effective; pub mod errors; pub mod models; pub mod project; diff --git a/crates/sqlbuild-config/src/models.rs b/crates/sqlbuild-config/src/models.rs index 8e84b2c66c..46e7882281 100644 --- a/crates/sqlbuild-config/src/models.rs +++ b/crates/sqlbuild-config/src/models.rs @@ -1,7 +1,7 @@ //! Configuration values with the value domain of Python's `tomllib` and PyYAML `safe_load`. use crate::constants::{ - FEBRUARY, MAX_HOUR, MAX_MICROSECOND, MAX_MINUTE, MAX_SECOND, MAX_YEAR, MIN_YEAR, + DECIMAL_BASE, FEBRUARY, MAX_HOUR, MAX_MICROSECOND, MAX_MINUTE, MAX_SECOND, MAX_YEAR, MIN_YEAR, MONTHS_PER_YEAR, }; @@ -99,3 +99,108 @@ impl ConfigValue { .map(|(_, value)| value) } } + +/// The content of one composed YAML node; aliases share node ids. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ComposedYamlContent { + Scalar(String), + Sequence(Vec), + Mapping(Vec<(usize, usize)>), +} + +impl ComposedYamlContent { + /// The value of a scalar node. + pub fn as_scalar(&self) -> Option<&str> { + match self { + Self::Scalar(value) => Some(value), + Self::Sequence(_) | Self::Mapping(_) => None, + } + } +} + +/// One node `yaml.compose` returns, with its PyYAML `start_mark.index` and `end_mark.index`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ComposedYamlNode { + pub content: ComposedYamlContent, + pub start: usize, + pub end: usize, +} + +/// The node graph PyYAML's safe `yaml.compose` returns; `root` is `None` for an empty stream. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ComposedYaml { + pub nodes: Vec, + pub root: Option, +} + +/// An exact non-negative integer: its decimal digit values, least significant first. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Natural { + digits: Vec, +} + +impl Natural { + /// The integer of decimal digit values, most significant first. + pub(crate) fn from_digit_values(values: impl DoubleEndedIterator) -> Self { + Self { + digits: values.rev().collect(), + } + .trimmed() + } + + fn trimmed(mut self) -> Self { + while self.digits.last() == Some(&0) { + self.digits.pop(); + } + self + } + + /// Whether the integer is zero. + pub(crate) fn is_zero(&self) -> bool { + self.digits.is_empty() + } + + /// `self * factor + addend`. + pub(crate) fn times_plus(&self, factor: u32, addend: u32) -> Self { + let base = u64::from(DECIMAL_BASE); + let mut carry = u64::from(addend); + let mut digits: Vec = Vec::with_capacity(self.digits.len() + 2); + for digit in &self.digits { + let value = u64::from(*digit) * u64::from(factor) + carry; + digits.push((value % base) as u32); + carry = value / base; + } + while carry > 0 { + digits.push((carry % base) as u32); + carry /= base; + } + Self { digits }.trimmed() + } + + /// `self + other`. + pub(crate) fn plus(&self, other: &Self) -> Self { + let mut carry = 0; + let mut digits: Vec = Vec::new(); + for index in 0..self.digits.len().max(other.digits.len()) { + let sum = self.digits.get(index).copied().unwrap_or_default() + + other.digits.get(index).copied().unwrap_or_default() + + carry; + digits.push(sum % DECIMAL_BASE); + carry = sum / DECIMAL_BASE; + } + digits.push(carry); + Self { digits }.trimmed() + } + + /// The decimal digits, `"0"` for zero. + pub(crate) fn decimal(&self) -> String { + if self.is_zero() { + return "0".to_owned(); + } + self.digits + .iter() + .rev() + .filter_map(|digit| char::from_digit(*digit, DECIMAL_BASE)) + .collect() + } +} diff --git a/crates/sqlbuild-config/src/project/_helpers/defaults.rs b/crates/sqlbuild-config/src/project/_helpers/defaults.rs new file mode 100644 index 0000000000..7516af9557 --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/defaults.rs @@ -0,0 +1,468 @@ +//! Loaders for `[defaults]`, `[path_defaults]` and `[materialization_defaults]`. + +use crate::models::ConfigValue; +use crate::project::_helpers::durations::parse_duration; +use crate::project::_helpers::fields::{ + bool_or_none, coerce_mapping, get, key_name, keyed, object, optional_batch_size, + optional_contract_policy, optional_cursor_start, optional_int, optional_mapping, + optional_non_negative_int, optional_nullable_bool, optional_strict_int, + optional_templated_bool, optional_text_field, string_sequence, value_or_none, +}; +use crate::project::_helpers::help::{change_policy_problem, change_policy_toml_help}; +use crate::project::_helpers::keys::{Place, reject_removed_virtual_keys, validate_allowed_keys}; +use crate::project::constants::{ + BATCH_CONCURRENCY_KEY, CHANGE_POLICY_KEYS, CONTRACT_POLICIES, DEFAULTS_KEYS, DISABLED_VALUE, + GLOB_SEGMENTS, INHERIT_VALUE, MATERIALIZATION_DEFAULT_KEYS, MAX_MICROBATCHES_KEY, + MODEL_HEADER_KEYS, MODELS_DIRECTORY_NAME, PROJECT_CONFIG_FILENAME, RETENTION_MATERIALIZATIONS, + TABLE_TYPES, TEMPLATE_OPEN_TOKEN, UNSUPPORTED_GLOB_MARKERS, ZERO_DAY_DURATION, +}; +use crate::project::errors::ConfigFailure; +use crate::project::models::{Built, ConfigFile}; +use crate::project::types::{Entries, Fields, Loaded}; +use sqlbuild_core::text::main::python_strip::python_strip; + +/// A retention or table-type field: where it lives and whether `'inherit'` is accepted. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct InheritableField<'a> { + pub(crate) key: &'a str, + pub(crate) label: &'a str, + pub(crate) allow_inherit: bool, +} + +fn place(file: ConfigFile, label: &str) -> Place<'_> { + Place { file, label } +} + +fn none() -> Built { + Built::Value(ConfigValue::Null) +} + +/// `_validate_change_policies` for one section of the project file. +fn validate_change_policies( + mapping: &Entries, + section: &str, + file: ConfigFile, +) -> Result<(), ConfigFailure> { + for key in CHANGE_POLICY_KEYS { + let Some(value) = get(mapping, key) else { + continue; + }; + if matches!(value, ConfigValue::String(text) if text.contains(TEMPLATE_OPEN_TOKEN)) { + continue; + } + if let Some(problem) = change_policy_problem(key, value)? { + return Err( + ConfigFailure::at(file, format!("[{section}] {problem}")).with_help( + change_policy_toml_help(key, PROJECT_CONFIG_FILENAME, section), + ), + ); + } + } + Ok(()) +} + +fn optional_text_fields(mapping: &Entries, keys: &[&'static str]) -> Loaded { + keys.iter() + .map(|key| keyed(*key, optional_text_field(mapping, key))) + .collect() +} + +fn counted(mapping: &Entries, key: &str) -> Loaded { + if get(mapping, key).is_none() { + return Ok(none()); + } + Ok(Built::Value(optional_int(mapping, key, 1)?)) +} + +fn cursor_fields(mapping: &Entries) -> Loaded { + let mut fields: Fields = vec![ + ( + "full_refresh", + bool_or_none(optional_nullable_bool(mapping, "full_refresh")?), + ), + ( + "append_cursor_inclusive", + optional_templated_bool(mapping, "append_cursor_inclusive")?, + ), + ( + "cursor_start", + optional_cursor_start(mapping, "cursor_start")?, + ), + ("cursor_end", optional_cursor_start(mapping, "cursor_end")?), + ]; + fields.extend(optional_text_fields( + mapping, + &[ + "cursor_start_max_ahead", + "cursor_start_max_action", + "cursor_future_max_distance", + "cursor_future_action", + "lookback", + ], + )?); + fields.push(("batch_size", optional_batch_size(mapping, "batch_size")?)); + fields.push(( + "batch_concurrency", + counted(mapping, BATCH_CONCURRENCY_KEY)?, + )); + fields.push(("max_microbatches", counted(mapping, MAX_MICROBATCHES_KEY)?)); + Ok(fields) +} + +fn sequences(mapping: &Entries, file: ConfigFile) -> Loaded<[Built; 3]> { + let load = |key: &str, label: &str| { + string_sequence(get(mapping, key), place(file, label)).map(Built::Tuple) + }; + Ok([ + load( + "row_diff_exclude_columns", + "defaults.row_diff_exclude_columns", + )?, + load("merge_exclude_columns", "defaults.merge_exclude_columns")?, + load("tags", "defaults.tags")?, + ]) +} + +/// `_load_defaults`. +pub(crate) fn load_defaults(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "defaults"))?; + reject_removed_virtual_keys(mapping, &["run_despite_unchanged"], place(file, "defaults"))?; + validate_allowed_keys(mapping, &DEFAULTS_KEYS, place(file, "[defaults]"))?; + validate_change_policies(mapping, "defaults", file)?; + let [row_diff_exclude, merge_exclude, tags] = sequences(mapping, file)?; + let tolerances = Built::Value(optional_mapping(mapping, "row_diff_tolerances")?); + let mut fields = optional_text_fields( + mapping, + &[ + "materialized", + "database", + "schema", + "seed_database", + "seed_schema", + "function_database", + "function_schema", + ], + )?; + fields.push(( + "contract", + optional_contract_policy(mapping, "contract", &CONTRACT_POLICIES)?, + )); + fields.extend(optional_text_fields( + mapping, + &[ + "incremental_strategy", + "incremental_mode", + "microbatch_strategy", + "cursor_watermark_mode", + ], + )?); + fields.push(("merge_exclude_columns", merge_exclude)); + fields.extend(cursor_fields(mapping)?); + fields.extend(optional_text_fields( + mapping, + &["unaccounted_partition_policy", "replay_on_change"], + )?); + fields.extend([ + ("row_diff_exclude_columns", row_diff_exclude), + ("row_diff_tolerances", tolerances), + ( + "row_diff_sample_rows", + value_or_none(optional_non_negative_int(mapping, "row_diff_sample_rows")?), + ), + ( + "row_diff_sample_seed", + value_or_none(optional_strict_int(mapping, "row_diff_sample_seed")?), + ), + ("tags", tags), + ]); + Ok(object("DefaultsConfig", fields)) +} + +fn invalid_path_key(file: ConfigFile, path_key: &str, problem: &str) -> ConfigFailure { + ConfigFailure::at(file, format!("path_defaults['{path_key}'] {problem}")) +} + +/// `_normalize_path_default_key`. +fn normalize_path_key(path_key: &str, file: ConfigFile) -> Loaded { + let normalized = python_strip(path_key); + if normalized.is_empty() { + return Err(ConfigFailure::at( + file, + "path_defaults contains an empty path key", + )); + } + if normalized.starts_with('/') { + return Err(invalid_path_key( + file, + path_key, + "is invalid. Use model-relative paths without a leading slash, for example \ + 'staging' or 'staging/nested'.", + )); + } + if normalized.ends_with('/') { + return Err(invalid_path_key( + file, + path_key, + "is invalid. Remove the trailing slash and use a model-relative path such as \ + 'staging' or 'staging/nested'.", + )); + } + let parts: Vec<&str> = normalized.split('/').collect(); + if parts.iter().any(|part| part.is_empty()) { + return Err(invalid_path_key( + file, + path_key, + "is invalid. Path defaults cannot contain empty path segments.", + )); + } + if parts[0] == MODELS_DIRECTORY_NAME { + return Err(invalid_path_key( + file, + path_key, + "uses redundant 'models/' prefix. Use a model-relative path such as 'staging' or \ + 'staging/nested'.", + )); + } + let unsupported: Vec<&str> = parts + .iter() + .copied() + .filter(|part| { + !GLOB_SEGMENTS.contains(part) && part.contains(UNSUPPORTED_GLOB_MARKERS.as_slice()) + }) + .collect(); + if !unsupported.is_empty() { + return Err(invalid_path_key( + file, + path_key, + &format!( + "contains unsupported glob segment(s): {}. Use '*' or '**' as complete path segments.", + unsupported.join(", ") + ), + )); + } + Ok(parts.join("/")) +} + +fn validate_path_default_tags( + entry: &Entries, + path_key: &str, + file: ConfigFile, +) -> Result<(), ConfigFailure> { + match get(entry, "tags") { + None => Ok(()), + Some(ConfigValue::List(items)) => { + if items + .iter() + .all(|item| matches!(item, ConfigValue::String(_))) + { + Ok(()) + } else { + Err(ConfigFailure::at( + file, + format!("path_defaults['{path_key}'].tags entries must be strings"), + )) + } + } + Some(_) => Err(ConfigFailure::at( + file, + format!("path_defaults['{path_key}'].tags must be a list"), + )), + } +} + +fn path_default_entry<'a>( + path_key: &str, + value: &'a ConfigValue, + file: ConfigFile, +) -> Loaded<&'a Entries> { + let ConfigValue::Map(entry) = value else { + return Err(invalid_path_key(file, path_key, "must be a mapping")); + }; + reject_removed_virtual_keys( + entry, + &["run_despite_unchanged"], + place(file, "path_defaults"), + )?; + let label = format!("path_defaults['{path_key}']"); + validate_allowed_keys(entry, &MODEL_HEADER_KEYS, place(file, &label))?; + validate_change_policies(entry, &format!("path_defaults.\"{path_key}\""), file)?; + validate_path_default_tags(entry, path_key, file)?; + Ok(entry) +} + +/// `_load_path_defaults`: entries by normalized model-relative path. +pub(crate) fn load_path_defaults(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "path_defaults"))?; + let mut loaded: Vec<(String, Built)> = Vec::new(); + for (key, value) in mapping { + let path_key = key_name(key); + let entry = path_default_entry(path_key, value, file)?; + let normalized = normalize_path_key(path_key, file)?; + if loaded.iter().any(|(existing, _)| *existing == normalized) { + return Err(ConfigFailure::at( + file, + format!( + "path_defaults defines both '{path_key}' and another key that normalize to \ + '{normalized}'. Use only the canonical model-relative form without a \ + leading 'models/' prefix." + ), + )); + } + loaded.push((normalized, Built::Value(ConfigValue::Map(entry.to_vec())))); + } + Ok(Built::Dict(loaded)) +} + +fn retention(desired_days: Option<&str>) -> Built { + match desired_days { + None => object( + "AuthoredTimeTravelRetention", + vec![("unmanaged", Built::Value(ConfigValue::Bool(true)))], + ), + Some(days) => object( + "AuthoredTimeTravelRetention", + vec![("desired_days", Built::Value(day_count(days)))], + ), + } +} + +fn day_count(days: &str) -> ConfigValue { + days.parse::().map_or_else( + |_| ConfigValue::BigInteger(days.to_owned()), + ConfigValue::Integer, + ) +} + +/// `_optional_retention_policy`. +pub(crate) fn optional_retention_policy( + mapping: &Entries, + field: InheritableField<'_>, + file: ConfigFile, +) -> Loaded { + let label = field.label; + let Some(value) = get(mapping, field.key) else { + return Ok(none()); + }; + let ConfigValue::String(text) = value else { + return Err(ConfigFailure::at( + file, + format!("{label} must be a whole-day string like '7d'"), + )); + }; + if field.allow_inherit && text == INHERIT_VALUE { + return Ok(none()); + } + if text == DISABLED_VALUE { + return Ok(retention(None)); + } + if text == ZERO_DAY_DURATION { + return Ok(retention(Some("0"))); + } + match parse_duration(text)?.filter(|duration| duration.is_whole_days()) { + Some(duration) => Ok(retention(Some(&duration.days()))), + None => Err(ConfigFailure::at( + file, + format!( + "{label} must be a whole-day string like '7d'{}", + if field.allow_inherit { + ", 'inherit', or 'disabled'" + } else { + " or 'disabled'" + } + ), + )), + } +} + +/// `_optional_table_type`. +pub(crate) fn optional_table_type( + mapping: &Entries, + field: InheritableField<'_>, + file: ConfigFile, +) -> Loaded { + match get(mapping, field.key) { + None => Ok(none()), + Some(ConfigValue::String(text)) if field.allow_inherit && text == INHERIT_VALUE => { + Ok(none()) + } + Some(ConfigValue::String(text)) if TABLE_TYPES.contains(&text.as_str()) => { + Ok(Built::Enum("TableType", text.clone())) + } + Some(_) => Err(ConfigFailure::at( + file, + format!( + "{} must be 'permanent' or 'transient'{}", + field.label, + if field.allow_inherit { + ", or 'inherit'" + } else { + "" + } + ), + )), + } +} + +fn materialization_defaults( + mapping: &Entries, + materialization: &str, + file: ConfigFile, +) -> Loaded { + let label = format!("materialization_defaults.{materialization}"); + let entry = coerce_mapping(get(mapping, materialization), place(file, &label))?; + validate_allowed_keys(entry, &MATERIALIZATION_DEFAULT_KEYS, place(file, &label))?; + let retention_label = format!("{label}.time_travel_retention"); + let table_type_label = format!("{label}.table_type"); + Ok(object( + "MaterializationRetentionDefaults", + vec![ + ( + "time_travel_retention", + optional_retention_policy( + entry, + InheritableField { + key: "time_travel_retention", + label: &retention_label, + allow_inherit: true, + }, + file, + )?, + ), + ( + "table_type", + optional_table_type( + entry, + InheritableField { + key: "table_type", + label: &table_type_label, + allow_inherit: true, + }, + file, + )?, + ), + ], + )) +} + +/// `_load_materialization_defaults`. +pub(crate) fn load_materialization_defaults( + payload: Option<&ConfigValue>, + file: ConfigFile, +) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "materialization_defaults"))?; + validate_allowed_keys( + mapping, + &RETENTION_MATERIALIZATIONS, + place(file, "materialization_defaults"), + )?; + let fields = RETENTION_MATERIALIZATIONS + .into_iter() + .map(|materialization| { + keyed( + materialization, + materialization_defaults(mapping, materialization, file), + ) + }) + .collect::>()?; + Ok(object("MaterializationDefaultsConfig", fields)) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs new file mode 100644 index 0000000000..827dfb0d08 --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -0,0 +1,413 @@ +//! Read the configuration documents and assemble the project and local configurations. + +use crate::errors::ConfigError; +use crate::models::ConfigValue; +use crate::project::_helpers::defaults::{ + load_defaults, load_materialization_defaults, load_path_defaults, +}; +use crate::project::_helpers::fields::{ + coerce_mapping, get, has, object, optional_mapping, optional_str, require_str, string_mapping, + text_or_none, +}; +use crate::project::_helpers::help::{ + Setting, SettingValue, join_helps, setting_help, setting_note, +}; +use crate::project::_helpers::keys::Place; +use crate::project::_helpers::sections::{ + load_constants, load_cost, load_cursors, load_dbt, load_diff, load_janitor, load_local_dbt, + load_microbatches, load_migrations, load_policy_flag, load_scenario, load_settings, + load_snapshots, setting_overrides, +}; +use crate::project::_helpers::sinks::load_sinks; +use crate::project::_helpers::targets::{ + load_connections, load_target_fields, local_target_config, project_target, +}; +use crate::project::_helpers::values::is_truthy; +use crate::project::constants::{ + LEGACY_EVENT_EXPORTERS_KEY, LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, + QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, SETTINGS_SECTION, SQL_ANALYSIS_KEY, +}; +use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; +use crate::project::models::{Built, ConfigFile, ConfigPaths}; +use crate::project::types::{Entries, Fields, Loaded}; +use crate::toml::main::load_toml::load_toml; +use std::path::Path; + +const PROJECT_FILE: ConfigFile = ConfigFile::Project; +const LOCAL_FILE: ConfigFile = ConfigFile::Local; + +/// The path of one configuration file. +pub(crate) fn path_of(paths: &ConfigPaths, file: ConfigFile) -> &Path { + match file { + ConfigFile::Project => &paths.project, + ConfigFile::LegacyProject => &paths.legacy_project, + ConfigFile::Local => &paths.local, + ConfigFile::LegacyLocal => &paths.legacy_local, + } +} + +/// The TOML file to read, rejecting a legacy YAML file that has no TOML replacement. +pub(crate) fn resolve(paths: &ConfigPaths, local: bool) -> Result { + let (file, legacy, name) = if local { + ( + ConfigFile::Local, + ConfigFile::LegacyLocal, + LOCAL_CONFIG_FILENAME, + ) + } else { + ( + ConfigFile::Project, + ConfigFile::LegacyProject, + PROJECT_CONFIG_FILENAME, + ) + }; + if path_of(paths, file).exists() || !path_of(paths, legacy).exists() { + return Ok(file); + } + Err(ConfigFailure::at( + legacy, + format!("is no longer supported. Rename it to {name} and convert the contents to TOML."), + )) +} + +fn invalid_toml(file: ConfigFile, error: &ConfigError) -> ConfigFailure { + let position = match (error.line, error.column) { + (Some(line), Some(column)) => format!(" (at line {line}, column {column})"), + (Some(line), None) => format!(" (at line {line})"), + _ => String::new(), + }; + ConfigFailure::at( + file, + format!("contains invalid TOML: {}{position}", error.message), + ) +} + +/// `_load_toml_mapping`: the top-level table of a TOML file. +pub(crate) fn load_toml_mapping( + paths: &ConfigPaths, + file: ConfigFile, +) -> Result { + let path = path_of(paths, file); + if !path.exists() { + return Err(LoadFailure::Config(ConfigFailure { + message: vec![ + MessagePiece::Text("Project config not found: ".to_owned()), + MessagePiece::File(file), + MessagePiece::Text( + ". Run sqb from a sqlbuild project directory or pass --project-dir.".to_owned(), + ), + ], + help: None, + class: FailureClass::ProjectConfig, + })); + } + let bytes = std::fs::read(path).map_err(|error| LoadFailure::Io(file, error))?; + let text = String::from_utf8(bytes).map_err(|error| LoadFailure::Utf8(error.into_bytes()))?; + load_toml(&text).map_err(|error| LoadFailure::Config(invalid_toml(file, &error))) +} + +fn flag(built: &Built, name: &str) -> Option { + let Built::Object(_, fields) = built else { + return None; + }; + fields + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| matches!(value, Built::Value(ConfigValue::Bool(true)))) +} + +fn tracking_setting(value: bool) -> Setting<'static> { + Setting { + file_name: PROJECT_CONFIG_FILENAME, + section: SETTINGS_SECTION, + key: QUERY_CHANGE_TRACKING_KEY, + value: SettingValue::Bool(value), + } +} + +fn reject_untracked_janitor(janitor: &Built, settings: &Built) -> Loaded<()> { + let enabled = flag(janitor, "enabled") == Some(true); + let tracked_only = flag(janitor, "delete_tracked_only") == Some(true); + if !enabled || !tracked_only || flag(settings, QUERY_CHANGE_TRACKING_KEY) != Some(false) { + return Ok(()); + } + Err(ConfigFailure::at( + PROJECT_FILE, + format!( + "[janitor] delete_tracked_only = true requires query change tracking; {}", + setting_note(&tracking_setting(false)) + ), + ) + .with_help(setting_help( + "to track which relations SQLBuild created", + &tracking_setting(true), + ))) +} + +fn identity(payload: &Entries) -> Loaded { + Ok(vec![ + ( + "name", + Built::Value(ConfigValue::String(require_str( + payload, + "name", + PROJECT_FILE, + )?)), + ), + ( + "adapter", + Built::Value(ConfigValue::String(require_str( + payload, + "adapter", + PROJECT_FILE, + )?)), + ), + ( + "default_target", + text_or_none(optional_str(payload, "default_target")?), + ), + ( + "connection", + Built::Value(optional_mapping(payload, "connection")?), + ), + ( + "connections", + load_connections(get(payload, "connections"), PROJECT_FILE)?, + ), + ]) +} + +fn sections(payload: &Entries) -> Loaded { + Ok(vec![ + ( + "settings", + load_settings(get(payload, "settings"), PROJECT_FILE)?, + ), + ( + "scopes", + load_policy_flag( + get(payload, "scopes"), + ("scopes", "enforce_placement", "ScopesConfig"), + PROJECT_FILE, + )?, + ), + ( + "references", + load_policy_flag( + get(payload, "references"), + ("references", "enforce_explicit", "ReferencesConfig"), + PROJECT_FILE, + )?, + ), + ("cost", load_cost(get(payload, "cost"), PROJECT_FILE)?), + ( + "constants", + load_constants(get(payload, "constants"), PROJECT_FILE)?, + ), + ( + "cursors", + load_cursors(get(payload, "cursors"), PROJECT_FILE)?, + ), + ( + "microbatches", + load_microbatches(get(payload, "microbatches"), PROJECT_FILE)?, + ), + ( + "defaults", + load_defaults(get(payload, "defaults"), PROJECT_FILE)?, + ), + ("diff", load_diff(get(payload, "diff"), PROJECT_FILE)?), + ( + "materialization_defaults", + load_materialization_defaults(get(payload, "materialization_defaults"), PROJECT_FILE)?, + ), + ( + "path_defaults", + load_path_defaults(get(payload, "path_defaults"), PROJECT_FILE)?, + ), + ("vars", string_mapping(get(payload, "vars"), PROJECT_FILE)?), + ]) +} + +fn targets(payload: &Entries) -> Loaded { + Ok(Built::Dict( + load_target_fields(get(payload, "targets"), PROJECT_FILE)? + .into_iter() + .map(|(name, fields)| (name, project_target(fields))) + .collect(), + )) +} + +fn operations(payload: &Entries) -> Loaded { + Ok(vec![ + ( + "janitor", + load_janitor(get(payload, "janitor"), PROJECT_FILE)?, + ), + ( + "migrations", + load_migrations(get(payload, "migrations"), PROJECT_FILE)?, + ), + ( + "snapshots", + load_snapshots(get(payload, "snapshots"), PROJECT_FILE)?, + ), + ( + "scenario", + load_scenario(get(payload, "scenario"), PROJECT_FILE)?, + ), + ("dbt", load_dbt(get(payload, "dbt"), PROJECT_FILE)?), + ("sinks", load_sinks(get(payload, "sinks"), PROJECT_FILE)?), + ]) +} + +fn project_fields(payload: &Entries) -> Loaded { + if get(payload, LEGACY_EVENT_EXPORTERS_KEY).is_some() { + return Err(ConfigFailure::at( + PROJECT_FILE, + "event_exporters was replaced by sinks.lifecycle", + )); + } + let mut fields = identity(payload)?; + fields.extend(sections(payload)?); + fields.push(("targets", targets(payload)?)); + let operations = operations(payload)?; + let settings = &fields[5].1; + reject_untracked_janitor(&operations[0].1, settings)?; + fields.extend(operations); + Ok(fields) +} + +/// `load_project_config`: the validated `ProjectConfig`. +pub(crate) fn project_config(paths: &ConfigPaths) -> Result { + let file = resolve(paths, false)?; + let table = load_toml_mapping(paths, file)?; + let payload = table.as_map().unwrap_or_default(); + Ok(object("ProjectConfig", project_fields(payload)?)) +} + +fn shared_policy_failure(value: &ConfigValue) -> ConfigFailure { + let help = join_helps(&[ + format!("remove {REQUIRE_SQL_ANALYSIS_KEY} from {LOCAL_CONFIG_FILENAME}"), + setting_help( + "to change the policy for everyone", + &Setting { + file_name: PROJECT_CONFIG_FILENAME, + section: SETTINGS_SECTION, + key: REQUIRE_SQL_ANALYSIS_KEY, + value: SettingValue::Bool(!is_truthy(value)), + }, + ), + ]); + ConfigFailure::at( + LOCAL_FILE, + format!( + "cannot set [settings] {REQUIRE_SQL_ANALYSIS_KEY}; it is a shared project policy \ + that only {PROJECT_CONFIG_FILENAME} can set" + ), + ) + .with_help(help) +} + +fn local_settings(payload: Option<&ConfigValue>) -> Loaded { + let mapping = coerce_mapping( + payload, + Place { + file: LOCAL_FILE, + label: "settings", + }, + )?; + if has(mapping, REQUIRE_SQL_ANALYSIS_KEY) { + let value = get(mapping, REQUIRE_SQL_ANALYSIS_KEY).unwrap_or(&ConfigValue::Null); + return Err(shared_policy_failure(value)); + } + let overrides = setting_overrides(mapping); + Ok(vec![ + ("settings", load_settings(payload, LOCAL_FILE)?), + ("setting_overrides", Built::FrozenSet(overrides)), + ]) +} + +fn local_fields(payload: &Entries) -> Loaded { + let mut fields: Fields = vec![ + ("target", text_or_none(optional_str(payload, "target")?)), + ("adapter", text_or_none(optional_str(payload, "adapter")?)), + ( + "connection", + Built::Value(optional_mapping(payload, "connection")?), + ), + ( + "connections", + load_connections(get(payload, "connections"), LOCAL_FILE)?, + ), + ]; + let targets: Vec<(String, Built)> = load_target_fields(get(payload, "targets"), LOCAL_FILE)? + .into_iter() + .map(|(name, target)| (name, local_target_config(target))) + .collect(); + fields.push(("targets", Built::Dict(targets))); + fields.extend(local_settings(get(payload, "settings"))?); + fields.push(("vars", string_mapping(get(payload, "vars"), LOCAL_FILE)?)); + fields.push(("dbt", load_local_dbt(get(payload, "dbt"), LOCAL_FILE)?)); + fields.push(( + "scenario", + load_scenario(get(payload, "scenario"), LOCAL_FILE)?, + )); + Ok(fields) +} + +/// `load_local_config`: the validated `LocalConfig`, or the default when the file is absent. +pub(crate) fn local_config(paths: &ConfigPaths) -> Result { + let file = resolve(paths, true)?; + if !path_of(paths, file).exists() { + return Ok(object("LocalConfig", Vec::new())); + } + let table = load_toml_mapping(paths, file)?; + let payload = table.as_map().unwrap_or_default(); + Ok(object("LocalConfig", local_fields(payload)?)) +} + +fn require_setting(value: bool) -> Setting<'static> { + Setting { + file_name: PROJECT_CONFIG_FILENAME, + section: SETTINGS_SECTION, + key: REQUIRE_SQL_ANALYSIS_KEY, + value: SettingValue::Bool(value), + } +} + +/// `validate_local_sql_analysis_policy` over (required, overridden, enabled) SQL analysis flags. +pub(crate) fn local_sql_analysis_failure( + paths: &ConfigPaths, + flags: (bool, bool, bool), +) -> Option { + let (required, overridden, enabled) = flags; + if !required || !overridden || enabled { + return None; + } + let file = match resolve(paths, true) { + Ok(file) => file, + Err(failure) => return Some(failure), + }; + let help = join_helps(&[ + format!( + "remove {SQL_ANALYSIS_KEY} from [settings] in {LOCAL_CONFIG_FILENAME}; to skip SQL \ + analysis for one run, use `--no-sql-analysis`" + ), + setting_help( + "to allow turning SQL analysis off locally", + &require_setting(false), + ), + ]); + Some( + ConfigFailure::at( + file, + format!( + "turns SQL analysis off with [settings] {SQL_ANALYSIS_KEY} = false, which this \ + project does not allow; {}", + setting_note(&require_setting(true)) + ), + ) + .with_help(help), + ) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/durations.rs b/crates/sqlbuild-config/src/project/_helpers/durations.rs new file mode 100644 index 0000000000..9880148e48 --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/durations.rs @@ -0,0 +1,99 @@ +//! `Duration.parse` and the duration properties the loaders check, over exact integers. + +use crate::models::Natural; +use crate::project::constants::PYTHON_INT_MAX_STR_DIGITS; +use crate::project::errors::ConfigFailure; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_decimal_value::python_decimal_value; + +const UNITS: [&str; 6] = ["y", "mo", "d", "h", "m", "s"]; +const DAY_INDEX: usize = 2; +const FIXED_UNIT_SECONDS: [u32; 4] = [86_400, 3_600, 60, 1]; + +/// A parsed non-zero duration. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Duration { + amounts: [Natural; 6], +} + +impl Duration { + /// `has_calendar_component`: years or months are non-zero. + pub(crate) fn has_calendar_component(&self) -> bool { + self.amounts[..DAY_INDEX] + .iter() + .any(|amount| !amount.is_zero()) + } + + /// `units == {"d"}`. + pub(crate) fn is_whole_days(&self) -> bool { + self.amounts + .iter() + .enumerate() + .all(|(index, amount)| (index == DAY_INDEX) != amount.is_zero()) + } + + /// The day amount in decimal digits. + pub(crate) fn days(&self) -> String { + self.amounts[DAY_INDEX].decimal() + } + + /// `fixed_seconds`: days and smaller units as whole seconds. + pub(crate) fn fixed_seconds(&self) -> Natural { + let mut total = Natural::default(); + for (amount, seconds) in self.amounts[DAY_INDEX..].iter().zip(FIXED_UNIT_SECONDS) { + total = total.plus(&amount.times_plus(seconds, 0)); + } + total + } +} + +/// The leading run of Python decimal digits as an integer, its byte length and its digit count. +fn leading_digits(text: &str) -> (Natural, usize, usize) { + let python = active_python_text(); + let mut values: Vec = Vec::new(); + let mut length = 0; + for character in text.chars() { + let Some(value) = python_decimal_value(python, character) else { + break; + }; + values.push(value); + length += character.len_utf8(); + } + let count = values.len(); + ( + Natural::from_digit_values(values.into_iter()), + length, + count, + ) +} + +/// `Duration.parse(text)`, `None` when it does not match or is zero; `int()`'s digit limit raises. +pub(crate) fn parse_duration(text: &str) -> Result, ConfigFailure> { + let mut rest: &str = text.strip_suffix('\n').unwrap_or(text); + let mut amounts: [Natural; 6] = Default::default(); + let mut counts: [usize; 6] = [0; 6]; + let mut next_unit = 0; + while !rest.is_empty() { + let (amount, length, count) = leading_digits(rest); + if length == 0 { + return Ok(None); + } + let Some(unit) = + (next_unit..UNITS.len()).find(|index| rest[length..].starts_with(UNITS[*index])) + else { + return Ok(None); + }; + amounts[unit] = amount; + counts[unit] = count; + rest = &rest[length + UNITS[unit].len()..]; + next_unit = unit + 1; + } + if let Some(count) = counts + .iter() + .find(|count| **count > PYTHON_INT_MAX_STR_DIGITS) + { + return Err(ConfigFailure::integer_digits(*count)); + } + let is_zero = amounts.iter().all(Natural::is_zero); + Ok((!is_zero).then_some(Duration { amounts })) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/fields.rs b/crates/sqlbuild-config/src/project/_helpers/fields.rs index af0621fe3c..4118588543 100644 --- a/crates/sqlbuild-config/src/project/_helpers/fields.rs +++ b/crates/sqlbuild-config/src/project/_helpers/fields.rs @@ -1,92 +1,318 @@ //! Typed reads of configuration fields with the Python loader's defaults and type checks. -use crate::errors::{ConfigError, ConfigErrorKind}; use crate::models::ConfigValue; +use crate::project::_helpers::keys::Place; +use crate::project::_helpers::values::{int_below, is_int}; +use crate::project::errors::ConfigFailure; +use crate::project::models::{Built, ConfigFile}; +use crate::project::types::{Entries, Loaded}; +use sqlbuild_core::text::main::python_strip::python_strip; -fn invalid(field: &str, expected: &str) -> ConfigError { - ConfigError::new( - ConfigErrorKind::InvalidField, - format!("{field} must be {expected}"), - ) +/// `mapping.get(key)`. +pub(crate) fn get<'a>(mapping: &'a Entries, key: &str) -> Option<&'a ConfigValue> { + mapping + .iter() + .find(|(candidate, _)| matches!(candidate, ConfigValue::String(text) if text == key)) + .map(|(_, value)| value) + .filter(|value| !matches!(value, ConfigValue::Null)) } -/// A section as a mapping; a missing section is empty, as in `_coerce_mapping`. -pub(crate) fn section<'value>( - table: &'value ConfigValue, - key: &str, -) -> Result<&'value [(ConfigValue, ConfigValue)], ConfigError> { - match table.get(key) { - None | Some(ConfigValue::Null) => Ok(&[]), - Some(value) => value.as_map().ok_or_else(|| invalid(key, "a mapping")), +/// `key in mapping`. +pub(crate) fn has(mapping: &Entries, key: &str) -> bool { + get(mapping, key).is_some() +} + +/// The text of a string key. +pub(crate) fn key_name(key: &ConfigValue) -> &str { + match key { + ConfigValue::String(text) => text, + _ => "", } } -/// A non-empty string, stripped, as in `_require_str`. -pub(crate) fn required_text(table: &ConfigValue, key: &str) -> Result { - optional_text(table, key)?.ok_or_else(|| invalid(key, "a non-empty string")) +/// `_coerce_mapping`: a missing value is empty; anything else must be a mapping. +pub(crate) fn coerce_mapping<'a>( + payload: Option<&'a ConfigValue>, + place: Place<'_>, +) -> Loaded<&'a Entries> { + match payload { + None => Ok(&[]), + Some(ConfigValue::Map(entries)) => Ok(entries), + Some(_) => Err(ConfigFailure::at( + place.file, + format!("{} must be a mapping", place.label), + )), + } } -/// An optional non-empty string, stripped, as in `_optional_str`. -pub(crate) fn optional_text(table: &ConfigValue, key: &str) -> Result, ConfigError> { - match table.get(key) { - None | Some(ConfigValue::Null) => Ok(None), - Some(ConfigValue::String(text)) if !text.trim().is_empty() => { - Ok(Some(text.trim().to_owned())) +fn non_empty(value: &ConfigValue) -> Option { + match value { + ConfigValue::String(text) if !python_strip(text).is_empty() => { + Some(python_strip(text).to_owned()) } - Some(_) => Err(invalid(key, "a non-empty string")), + _ => None, } } -/// An optional boolean with a default, as in `_optional_bool`. -pub(crate) fn flag( - entries: &[(ConfigValue, ConfigValue)], - key: &str, - default: bool, -) -> Result { - match lookup(entries, key) { - None | Some(ConfigValue::Null) => Ok(default), - Some(ConfigValue::Bool(value)) => Ok(*value), - Some(_) => Err(invalid(key, "a boolean")), +/// `_optional_str`: a stripped non-empty string, or `None` when absent. +pub(crate) fn optional_str(mapping: &Entries, key: &str) -> Loaded> { + let Some(value) = get(mapping, key) else { + return Ok(None); + }; + non_empty(value).map(Some).ok_or_else(|| { + ConfigFailure::plain(format!( + "Expected '{key}' to be a non-empty string when provided" + )) + }) +} + +/// `_optional_str` as a field value. +pub(crate) fn optional_text_field(mapping: &Entries, key: &str) -> Loaded { + Ok(text_or_none(optional_str(mapping, key)?)) +} + +/// A string, or `None`. +pub(crate) fn text_or_none(value: Option) -> Built { + Built::Value(value.map_or(ConfigValue::Null, ConfigValue::String)) +} + +/// `_require_str`. +pub(crate) fn require_str(mapping: &Entries, key: &str, file: ConfigFile) -> Loaded { + get(mapping, key) + .and_then(non_empty) + .ok_or_else(|| ConfigFailure::at(file, format!("must define non-empty string '{key}'"))) +} + +/// `_optional_mapping`: the authored mapping, or an empty one. +pub(crate) fn optional_mapping(mapping: &Entries, key: &str) -> Loaded { + match get(mapping, key) { + None => Ok(ConfigValue::Map(Vec::new())), + Some(value @ ConfigValue::Map(_)) => Ok(value.clone()), + Some(_) => Err(ConfigFailure::plain(format!( + "Expected '{key}' to be a mapping when provided" + ))), + } +} + +fn expected(key: &str, kind: &str) -> ConfigFailure { + ConfigFailure::plain(format!("Expected '{key}' to be {kind} when provided")) +} + +/// `_optional_nullable_bool`. +pub(crate) fn optional_nullable_bool(mapping: &Entries, key: &str) -> Loaded> { + match get(mapping, key) { + None => Ok(None), + Some(ConfigValue::Bool(flag)) => Ok(Some(*flag)), + Some(_) => Err(expected(key, "a boolean")), + } +} + +/// `_optional_bool`. +pub(crate) fn optional_bool(mapping: &Entries, key: &str, default: bool) -> Loaded { + Ok(optional_nullable_bool(mapping, key)?.unwrap_or(default)) +} + +/// A boolean, or `None`. +pub(crate) fn bool_or_none(value: Option) -> Built { + Built::Value(value.map_or(ConfigValue::Null, ConfigValue::Bool)) +} + +/// `_optional_templated_bool`: a boolean or a template string. +pub(crate) fn optional_templated_bool(mapping: &Entries, key: &str) -> Loaded { + match get(mapping, key) { + None => Ok(Built::Value(ConfigValue::Null)), + Some(value @ (ConfigValue::Bool(_) | ConfigValue::String(_))) => { + Ok(Built::Value(value.clone())) + } + Some(_) => Err(expected(key, "a boolean")), + } +} + +/// `_optional_nullable_int`: any Python integer, booleans included. +pub(crate) fn optional_nullable_int(mapping: &Entries, key: &str) -> Loaded> { + match get(mapping, key) { + None => Ok(None), + Some(value) if is_int(value) => Ok(Some(value.clone())), + Some(_) => Err(expected(key, "an integer")), + } +} + +/// `_optional_int`. +pub(crate) fn optional_int(mapping: &Entries, key: &str, default: i64) -> Loaded { + Ok(optional_nullable_int(mapping, key)?.unwrap_or(ConfigValue::Integer(default))) +} + +/// `_optional_strict_int`: an integer that is not a boolean. +pub(crate) fn optional_strict_int(mapping: &Entries, key: &str) -> Loaded> { + if matches!(get(mapping, key), Some(ConfigValue::Bool(_))) { + return Err(expected(key, "an integer")); } + optional_nullable_int(mapping, key) } -/// The value of a string key among mapping entries. -pub(crate) fn lookup<'value>( - entries: &'value [(ConfigValue, ConfigValue)], +/// `_optional_non_negative_int`. +pub(crate) fn optional_non_negative_int( + mapping: &Entries, key: &str, -) -> Option<&'value ConfigValue> { - entries - .iter() - .find(|(candidate, _)| matches!(candidate, ConfigValue::String(text) if text == key)) - .map(|(_, value)| value) +) -> Loaded> { + let value = optional_strict_int(mapping, key)?; + if value.as_ref().is_some_and(|number| int_below(number, 0)) { + return Err(ConfigFailure::plain(format!( + "Expected '{key}' to be zero or greater when provided" + ))); + } + Ok(value) } -/// The text of a mapping key; TOML keys are always strings. -pub(crate) fn key_text(key: &ConfigValue) -> Result { - match key { - ConfigValue::String(text) => Ok(text.clone()), - _ => Err(invalid("a key", "a string")), +/// An optional value as a field. +pub(crate) fn value_or_none(value: Option) -> Built { + Built::Value(value.unwrap_or(ConfigValue::Null)) +} + +/// `_optional_scalar_batch_size`: a string or an integer. +pub(crate) fn optional_batch_size(mapping: &Entries, key: &str) -> Loaded { + match get(mapping, key) { + None => Ok(Built::Value(ConfigValue::Null)), + Some(value @ ConfigValue::String(_)) => Ok(Built::Value(value.clone())), + Some(value) if is_int(value) => Ok(Built::Value(value.clone())), + Some(_) => Err(expected(key, "a string or integer")), } } -/// String values, as in `_load_string_mapping`. -pub(crate) fn text_mapping( - table: &ConfigValue, +/// `_optional_cursor_start`: a string, integer, date or datetime. +pub(crate) fn optional_cursor_start(mapping: &Entries, key: &str) -> Loaded { + match get(mapping, key) { + None => Ok(Built::Value(ConfigValue::Null)), + Some( + value @ (ConfigValue::String(_) | ConfigValue::Date(_) | ConfigValue::DateTime(_)), + ) => Ok(Built::Value(value.clone())), + Some(value) if is_int(value) => Ok(Built::Value(value.clone())), + Some(_) => Err(expected(key, "a string, integer, or date-like value")), + } +} + +/// A string value from a fixed vocabulary, or `default` when absent. +pub(crate) fn optional_choice( + mapping: &Entries, key: &str, -) -> Result, ConfigError> { - section(table, key)? + choices: &[&str], + default: &str, +) -> Loaded { + match get(mapping, key) { + None => Ok(default.to_owned()), + Some(ConfigValue::String(text)) if choices.contains(&text.as_str()) => Ok(text.clone()), + Some(ConfigValue::String(_)) => Err(ConfigFailure::plain(format!( + "Expected '{key}' to be one of: {}", + choices.join(", ") + ))), + Some(_) => Err(expected(key, "a string")), + } +} + +/// `_optional_contract_policy`. +pub(crate) fn optional_contract_policy( + mapping: &Entries, + key: &str, + policies: &[&str], +) -> Loaded { + match get(mapping, key) { + None => Ok(Built::Value(ConfigValue::Null)), + Some(value @ ConfigValue::String(text)) if policies.contains(&text.as_str()) => { + Ok(Built::Value(value.clone())) + } + Some(ConfigValue::String(_)) => Err(ConfigFailure::plain( + "Expected 'contract' to be one of: enforced, none", + )), + Some(_) => Err(expected(key, "a string")), + } +} + +/// `_load_string_sequence`: a list of strings, empty when absent. +pub(crate) fn string_sequence( + payload: Option<&ConfigValue>, + place: Place<'_>, +) -> Loaded> { + let items = match payload { + None => return Ok(Vec::new()), + Some(ConfigValue::List(items)) => items, + Some(_) => { + return Err(ConfigFailure::at( + place.file, + format!("{} must be a list of strings", place.label), + )); + } + }; + items .iter() - .map(|(name, value)| match value { - ConfigValue::String(text) => Ok((key_text(name)?, text.clone())), - _ => Err(invalid(&format!("{key}.{}", key_text(name)?), "a string")), + .map(|item| match item { + ConfigValue::String(text) => Ok(text.clone()), + _ => Err(ConfigFailure::at( + place.file, + format!("{} must contain only strings", place.label), + )), }) .collect() } -/// The keys of a section, in document order. -pub(crate) fn section_keys(table: &ConfigValue, key: &str) -> Result, ConfigError> { - section(table, key)? +/// `_load_string_mapping`: string values by key. +pub(crate) fn string_mapping(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping( + payload, + Place { + file, + label: "mapping", + }, + )?; + mapping .iter() - .map(|(name, _)| key_text(name)) - .collect() + .map(|(key, value)| match value { + ConfigValue::String(_) => Ok((key_name(key).to_owned(), Built::Value(value.clone()))), + _ => Err(ConfigFailure::at( + file, + format!("expected string value for '{}'", key_name(key)), + )), + }) + .collect::>>() + .map(Built::Dict) +} + +/// `_load_object_mapping`: the authored mapping, or an empty one. +pub(crate) fn object_mapping(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping( + payload, + Place { + file, + label: "mapping", + }, + )?; + Ok(Built::Value(ConfigValue::Map(mapping.to_vec()))) +} + +/// `entries[key] = value`: a repeated key keeps its first position and takes the new value. +pub(crate) fn insert_entry( + entries: Vec<(String, Built)>, + key: String, + value: Built, +) -> Vec<(String, Built)> { + let mut updated = entries; + match updated.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => updated.push((key, value)), + } + updated +} + +/// A dataclass instance. +pub(crate) fn object(class: &'static str, fields: Vec<(&'static str, Built)>) -> Built { + Built::Object(class, fields) +} + +/// A `StrEnum` member by value. +pub(crate) fn member(class: &'static str, value: &str) -> Built { + Built::Enum(class, value.to_owned()) +} + +/// A loaded value paired with its key. +pub(crate) fn keyed(key: K, loaded: Loaded) -> Loaded<(K, T)> { + loaded.map(|value| (key, value)) } diff --git a/crates/sqlbuild-config/src/project/_helpers/files.rs b/crates/sqlbuild-config/src/project/_helpers/files.rs deleted file mode 100644 index c34d90dd4e..0000000000 --- a/crates/sqlbuild-config/src/project/_helpers/files.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Locate and load configuration files as the Python loader does. - -use crate::errors::{ConfigError, ConfigErrorKind}; -use crate::models::ConfigValue; -use crate::toml::main::load_toml::load_toml; -use std::path::Path; - -/// Reject a legacy YAML file that has no TOML replacement, as the Python loader does. -pub(crate) fn reject_legacy( - directory: &Path, - filename: &str, - legacy_filename: &str, -) -> Result<(), ConfigError> { - if !directory.join(filename).exists() && directory.join(legacy_filename).exists() { - return Err(ConfigError::new( - ConfigErrorKind::InvalidField, - format!("{legacy_filename} is no longer supported; rename it to {filename}"), - )); - } - Ok(()) -} - -/// Read and parse one TOML file the way `tomllib.load` does. -pub(crate) fn load_toml_file(path: &Path) -> Result { - let bytes = std::fs::read(path).map_err(|error| { - ConfigError::new( - ConfigErrorKind::Missing, - format!("{}: {error}", path.display()), - ) - })?; - let text = String::from_utf8(bytes).map_err(|error| { - ConfigError::new(ConfigErrorKind::Syntax, format!("invalid UTF-8: {error}")) - })?; - load_toml(&text) -} diff --git a/crates/sqlbuild-config/src/project/_helpers/help.rs b/crates/sqlbuild-config/src/project/_helpers/help.rs new file mode 100644 index 0000000000..1c5160645a --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/help.rs @@ -0,0 +1,129 @@ +//! Help text naming an exact setting, and change-policy value checks. + +use crate::models::ConfigValue; +use crate::project::_helpers::durations::parse_duration; +use crate::project::constants::{ + ADDITIONAL_HELP_SEPARATOR, BOUNDED_REPLAY_PREFIX, ON_SCHEMA_CHANGE_EXAMPLE, + ON_SCHEMA_CHANGE_KEY, ON_SCHEMA_CHANGE_VALUES, REPLAY_ON_CHANGE_EXAMPLE, + REPLAY_ON_CHANGE_VALUES, SETTING_SNIPPET_INDENT, +}; +use crate::project::errors::ConfigFailure; +use sqlbuild_core::text::main::python_strip::python_strip; + +/// A setting value as TOML shows it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SettingValue<'a> { + Bool(bool), + Text(&'a str), +} + +/// One setting: the file, section and key it lives at and the value to show. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Setting<'a> { + pub(crate) file_name: &'a str, + pub(crate) section: &'a str, + pub(crate) key: &'a str, + pub(crate) value: SettingValue<'a>, +} + +fn toml_value(value: SettingValue<'_>) -> String { + match value { + SettingValue::Bool(true) => "true".to_owned(), + SettingValue::Bool(false) => "false".to_owned(), + SettingValue::Text(text) => { + format!("\"{}\"", text.replace('\\', "\\\\").replace('"', "\\\"")) + } + } +} + +/// `setting_help`: `, set this in :` and the TOML section and key. +pub(crate) fn setting_help(purpose: &str, setting: &Setting<'_>) -> String { + format!( + "{purpose}, set this in {}:\n{SETTING_SNIPPET_INDENT}[{}]\n{SETTING_SNIPPET_INDENT}{} = {}", + setting.file_name, + setting.section, + setting.key, + toml_value(setting.value) + ) +} + +/// `setting_note` for a value a file sets explicitly. +pub(crate) fn setting_note(setting: &Setting<'_>) -> String { + format!( + "{} sets [{}] {} = {}", + setting.file_name, + setting.section, + setting.key, + toml_value(setting.value) + ) +} + +/// `join_helps`: one `= help:` label per paragraph. +pub(crate) fn join_helps(helps: &[String]) -> String { + helps.join(ADDITIONAL_HELP_SEPARATOR) +} + +fn valid_values(key: &str) -> String { + if key == ON_SCHEMA_CHANGE_KEY { + ON_SCHEMA_CHANGE_VALUES.join(", ") + } else { + REPLAY_ON_CHANGE_VALUES.join(", ") + } +} + +fn replay_problem(key: &str, value: &str) -> Result, ConfigFailure> { + let Some(bounded) = value.strip_prefix(BOUNDED_REPLAY_PREFIX) else { + return Ok(None); + }; + let duration = python_strip(bounded); + if parse_duration(duration)?.is_some() { + return Ok(None); + } + Ok(Some(format!( + "{key} '{value}' has an invalid duration '{duration}'; use a positive duration \ + such as 14d, 12h or 1mo" + ))) +} + +/// `change_policy_problem`: why an `on_schema_change` or `replay_on_change` value is invalid. +pub(crate) fn change_policy_problem( + key: &str, + value: &ConfigValue, +) -> Result, ConfigFailure> { + let valid = valid_values(key); + let ConfigValue::String(text) = value else { + return Ok(Some(format!( + "{key} must be a string; valid values: {valid}" + ))); + }; + if key == ON_SCHEMA_CHANGE_KEY && ON_SCHEMA_CHANGE_VALUES.contains(&text.as_str()) { + return Ok(None); + } + if key != ON_SCHEMA_CHANGE_KEY && text.starts_with(BOUNDED_REPLAY_PREFIX) { + return replay_problem(key, text); + } + if key != ON_SCHEMA_CHANGE_KEY && REPLAY_ON_CHANGE_VALUES[..2].contains(&text.as_str()) { + return Ok(None); + } + Ok(Some(format!( + "unknown {key} '{text}'; valid values: {valid}" + ))) +} + +/// `change_policy_toml_help`: a valid TOML line for the change-policy key. +pub(crate) fn change_policy_toml_help(key: &str, file_name: &str, section: &str) -> String { + let example = if key == ON_SCHEMA_CHANGE_KEY { + ON_SCHEMA_CHANGE_EXAMPLE + } else { + REPLAY_ON_CHANGE_EXAMPLE + }; + setting_help( + &format!("use a valid {key}"), + &Setting { + file_name, + section, + key, + value: SettingValue::Text(example), + }, + ) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/keys.rs b/crates/sqlbuild-config/src/project/_helpers/keys.rs new file mode 100644 index 0000000000..7d06076c1a --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/keys.rs @@ -0,0 +1,133 @@ +//! Rejection of unknown and removed keys in configuration mappings. + +use crate::models::ConfigValue; +use crate::project::constants::{LISTED_SUPPORTED_KEYS_LIMIT, SUGGESTION_CUTOFF}; +use crate::project::errors::ConfigFailure; +use crate::project::models::ConfigFile; +use crate::project::types::{Entries, Loaded}; +use sqlbuild_core::text::main::close_matches::close_matches; + +/// Where a mapping lives: its file and the label messages name it by. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Place<'a> { + pub(crate) file: ConfigFile, + pub(crate) label: &'a str, +} + +fn sorted_unique<'a>(keys: impl IntoIterator) -> Vec<&'a str> { + let mut sorted: Vec<&str> = keys.into_iter().collect(); + sorted.sort_unstable(); + sorted.dedup(); + sorted +} + +fn key_text(key: &ConfigValue) -> &str { + match key { + ConfigValue::String(text) => text, + _ => "", + } +} + +fn has_key(mapping: &Entries, key: &str) -> bool { + mapping + .iter() + .any(|(candidate, _)| key_text(candidate) == key) +} + +fn unknown_keys<'a>(mapping: &'a Entries, allowed: &[&str]) -> Vec<&'a str> { + sorted_unique( + mapping + .iter() + .map(|(key, _)| key_text(key)) + .filter(|key| !allowed.contains(key)), + ) +} + +/// `unsupported_keys_help`: the nearest supported key for each unsupported key. +pub(crate) fn unsupported_keys_help(keys: &[&str], supported: &[&str]) -> String { + let candidates = sorted_unique(supported.iter().copied()); + let suggestions: Vec = keys + .iter() + .filter_map(|key| { + let nearest = close_matches(key, &candidates, 1, SUGGESTION_CUTOFF) + .into_iter() + .next()?; + Some(if keys.len() == 1 { + format!("'{nearest}'") + } else { + format!("'{nearest}' for '{key}'") + }) + }) + .collect(); + if !suggestions.is_empty() { + return format!("did you mean {}?", suggestions.join(", ")); + } + if candidates.len() <= LISTED_SUPPORTED_KEYS_LIMIT { + return format!("supported keys: {}", candidates.join(", ")); + } + "remove the key; see the reference for supported keys".to_owned() +} + +/// `_validate_allowed_keys`: reject keys outside `allowed`, listing the allowed keys. +pub(crate) fn validate_allowed_keys( + mapping: &Entries, + allowed: &[&str], + place: Place<'_>, +) -> Result<(), ConfigFailure> { + let unknown = unknown_keys(mapping, allowed); + if unknown.is_empty() { + return Ok(()); + } + let listed = sorted_unique(allowed.iter().copied()).join(", "); + Err(ConfigFailure::at( + place.file, + format!( + "{} contains unknown key(s): {}. Allowed keys: {listed}", + place.label, + unknown.join(", ") + ), + ) + .with_help(unsupported_keys_help(&unknown, allowed))) +} + +/// `reject_unknown_mapping_keys`: the same check with the shared YAML-section wording. +pub(crate) fn reject_unknown_mapping_keys( + mapping: &Entries, + allowed: &[&str], + place: Place<'_>, +) -> Loaded<()> { + let unknown = unknown_keys(mapping, allowed); + if unknown.is_empty() { + return Ok(()); + } + let listed = sorted_unique(allowed.iter().copied()).join(", "); + Err(ConfigFailure::at( + place.file, + format!( + "{} has unknown keys: {}; allowed keys: {listed}", + place.label, + unknown.join(", ") + ), + ) + .with_help(unsupported_keys_help(&unknown, allowed))) +} + +/// `_reject_removed_virtual_keys`: options removed with virtual environments. +pub(crate) fn reject_removed_virtual_keys( + mapping: &Entries, + removed: &[&str], + place: Place<'_>, +) -> Loaded<()> { + let present = sorted_unique(removed.iter().copied().filter(|key| has_key(mapping, key))); + if present.is_empty() { + return Ok(()); + } + Err(ConfigFailure::at( + place.file, + format!( + "[{}] option(s) {} were removed with virtual environments; projects run in direct mode", + place.label, + present.join(", ") + ), + )) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/mod.rs b/crates/sqlbuild-config/src/project/_helpers/mod.rs index b21adf99f8..90fd012f57 100644 --- a/crates/sqlbuild-config/src/project/_helpers/mod.rs +++ b/crates/sqlbuild-config/src/project/_helpers/mod.rs @@ -1,2 +1,10 @@ +pub(crate) mod defaults; +pub(crate) mod documents; +pub(crate) mod durations; pub(crate) mod fields; -pub(crate) mod files; +pub(crate) mod help; +pub(crate) mod keys; +pub(crate) mod sections; +pub(crate) mod sinks; +pub(crate) mod targets; +pub(crate) mod values; diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs new file mode 100644 index 0000000000..d0c10782ff --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -0,0 +1,682 @@ +//! Loaders for the scalar sections of the project and local configuration files. + +use crate::models::ConfigValue; +use crate::project::_helpers::durations::parse_duration; +use crate::project::_helpers::fields::{ + coerce_mapping, get, has, insert_entry, key_name, keyed, member, object, object_mapping, + optional_bool, optional_choice, optional_int, optional_nullable_int, optional_str, + optional_text_field, string_mapping, string_sequence, text_or_none, value_or_none, +}; +use crate::project::_helpers::keys::{Place, reject_removed_virtual_keys, validate_allowed_keys}; +use crate::project::_helpers::values::{int_below, python_equal}; +use crate::project::constants::{ + AUDIT_SEVERITIES, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, + DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_QUERY_ARTIFACT_TTL, + FUTURE_CURSOR_ACTIONS, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, + MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, + REQUIRE_SQL_ANALYSIS_KEY, RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, + SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, + SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, + USD_PER_CREDIT_KEY, ZERO_DAY_DURATION, ZERO_FIXED_DURATION, ZERO_FLOAT, +}; +use crate::project::errors::ConfigFailure; +use crate::project::models::{Built, ConfigFile}; +use crate::project::types::{Entries, Fields, Loaded}; +use sqlbuild_core::text::main::python_strip::python_strip; + +fn place(file: ConfigFile, label: &str) -> Place<'_> { + Place { file, label } +} + +/// `_fixed_duration_seconds`: whole seconds of a fixed duration, or `None`. +pub(crate) fn fixed_duration_seconds(value: &str, allow_zero: bool) -> Loaded> { + if allow_zero && value == ZERO_FIXED_DURATION { + return Ok(Some("0".to_owned())); + } + Ok(parse_duration(value)? + .filter(|duration| !duration.has_calendar_component()) + .map(|duration| duration.fixed_seconds().decimal())) +} + +fn settings_mapping(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded<&Entries> { + let mapping = coerce_mapping(payload, place(file, "settings"))?; + reject_removed_virtual_keys( + mapping, + &["virtual_environments", "changes_only"], + place(file, "settings"), + )?; + let allowed: Vec<&str> = SETTINGS_KEYS + .into_iter() + .chain([LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY]) + .collect(); + validate_allowed_keys(mapping, &allowed, place(file, "settings"))?; + if has(mapping, CONCURRENCY_KEY) && has(mapping, LEGACY_CONCURRENCY_KEY) { + return Err(ConfigFailure::plain( + "settings cannot define both 'concurrency' and legacy 'max_concurrency'", + )); + } + if let (Some(current), Some(legacy)) = ( + get(mapping, SQL_ANALYSIS_KEY), + get(mapping, LEGACY_SQL_VALIDATION_KEY), + ) && !python_equal(current, legacy) + { + return Err(ConfigFailure::plain( + "settings.sql_analysis conflicts with legacy settings.sql_validation", + )); + } + Ok(mapping) +} + +fn column_contract_mode(mapping: &Entries) -> Loaded { + let mode = match get(mapping, "column_contract_mode") { + None => Some(COLUMN_CONTRACT_MODES[0]), + Some(ConfigValue::String(text)) => Some(text.as_str()), + Some(_) => None, + }; + match mode { + Some(text) if COLUMN_CONTRACT_MODES.contains(&text) => { + Ok(member("ColumnContractMode", text)) + } + _ => Err(ConfigFailure::plain( + "settings.column_contract_mode must be one of: implicit, explicit", + )), + } +} + +fn unaccounted_partition_policy(mapping: &Entries) -> Loaded { + let policy = optional_str(mapping, "microbatch_unaccounted_partition_policy")? + .unwrap_or_else(|| "synthesize".to_owned()); + if !UNACCOUNTED_PARTITION_POLICIES.contains(&policy.as_str()) { + return Err(ConfigFailure::plain(format!( + "settings.microbatch_unaccounted_partition_policy must be one of: {}", + UNACCOUNTED_PARTITION_POLICIES.join(", ") + ))); + } + Ok(policy) +} + +fn concurrency(mapping: &Entries) -> Loaded { + let key = if has(mapping, LEGACY_CONCURRENCY_KEY) { + LEGACY_CONCURRENCY_KEY + } else { + CONCURRENCY_KEY + }; + let value = optional_int(mapping, key, 1)?; + if int_below(&value, 1) { + return Err(ConfigFailure::plain("settings.concurrency must be >= 1")); + } + Ok(value) +} + +fn default_audit_severity(mapping: &Entries) -> Loaded { + match optional_str(mapping, "default_audit_severity")? { + None => Ok(Built::Value(ConfigValue::Null)), + Some(severity) if AUDIT_SEVERITIES.contains(&severity.as_str()) => { + Ok(member("AuditSeverity", &severity)) + } + Some(_) => Err(ConfigFailure::plain(format!( + "settings.default_audit_severity must be one of: {}", + AUDIT_SEVERITIES.join(", ") + ))), + } +} + +/// `_load_settings`. +pub(crate) fn load_settings(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = settings_mapping(payload, file)?; + let analysis_key = if has(mapping, SQL_ANALYSIS_KEY) { + SQL_ANALYSIS_KEY + } else { + LEGACY_SQL_VALIDATION_KEY + }; + let fields: Fields = vec![ + ( + "sql_analysis", + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + analysis_key, + true, + )?)), + ), + ( + "require_sql_analysis", + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + REQUIRE_SQL_ANALYSIS_KEY, + false, + )?)), + ), + ( + "query_change_tracking", + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + QUERY_CHANGE_TRACKING_KEY, + true, + )?)), + ), + ("column_contract_mode", column_contract_mode(mapping)?), + ( + "auto_load_sources", + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + "auto_load_sources", + true, + )?)), + ), + ( + "microbatch_concurrency", + Built::Value(ConfigValue::Bool(optional_bool( + mapping, + "microbatch_concurrency", + false, + )?)), + ), + ( + "microbatch_unaccounted_partition_policy", + Built::Value(ConfigValue::String(unaccounted_partition_policy(mapping)?)), + ), + ("concurrency", Built::Value(concurrency(mapping)?)), + ( + "table_promotion_mode", + text_or_none(optional_str(mapping, "table_promotion_mode")?), + ), + ("default_audit_severity", default_audit_severity(mapping)?), + ( + "default_audit_run_scope", + text_or_none(optional_str(mapping, "default_audit_run_scope")?), + ), + ]; + Ok(object("SettingsConfig", fields)) +} + +/// The `[settings]` keys a local file sets, under their current names. +pub(crate) fn setting_overrides(mapping: &Entries) -> Vec { + let mut overrides: Vec = Vec::new(); + for (key, _) in mapping { + let name = match key_name(key) { + LEGACY_CONCURRENCY_KEY => CONCURRENCY_KEY, + LEGACY_SQL_VALIDATION_KEY => SQL_ANALYSIS_KEY, + other if SETTINGS_KEYS.contains(&other) => other, + _ => continue, + }; + if !overrides.iter().any(|existing| existing == name) { + overrides.push(name.to_owned()); + } + } + overrides +} + +fn cursor_limit( + mapping: &Entries, + keys: (&str, &str), + label: &str, +) -> Loaded<(Option, Built)> { + let limit = optional_str(mapping, keys.0)?; + let invalid = match limit.as_deref() { + Some(text) if text != ZERO_DAY_DURATION => parse_duration(text)?.is_none(), + _ => false, + }; + if invalid { + return Err(ConfigFailure::plain(format!( + "{label}.{} must be a duration like 0d, 7d, 12h, or 1mo", + keys.0 + ))); + } + let action = optional_str(mapping, keys.1)?; + let action = action.as_deref().unwrap_or("error"); + if !FUTURE_CURSOR_ACTIONS.contains(&action) { + return Err(ConfigFailure::plain(format!( + "{label}.action must be one of: cap, error" + ))); + } + Ok((limit, member("FutureCursorAction", action))) +} + +/// `_load_cursors`. +pub(crate) fn load_cursors(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "cursors"))?; + validate_allowed_keys(mapping, &["future", "start"], place(file, "cursors"))?; + let future = coerce_mapping(get(mapping, "future"), place(file, "cursors.future"))?; + validate_allowed_keys( + future, + &["max_distance", "action"], + place(file, "cursors.future"), + )?; + let (max_distance, action) = + cursor_limit(future, ("max_distance", "action"), "cursors.future")?; + let start = coerce_mapping(get(mapping, "start"), place(file, "cursors.start"))?; + validate_allowed_keys( + start, + &["max_ahead", "action"], + place(file, "cursors.start"), + )?; + let (max_ahead, start_action) = cursor_limit(start, ("max_ahead", "action"), "cursors.start")?; + Ok(object( + "CursorsConfig", + vec![ + ( + "future", + object( + "FutureCursorsConfig", + vec![ + ("max_distance", text_or_none(max_distance)), + ("action", action), + ], + ), + ), + ( + "start", + object( + "StartCursorsConfig", + vec![ + ("max_ahead", text_or_none(max_ahead)), + ("action", start_action), + ], + ), + ), + ], + )) +} + +fn positive_max_batches(limits: &Entries) -> Loaded> { + if !has(limits, MAX_BATCHES_KEY) { + return Ok(None); + } + let invalid = + || ConfigFailure::plain("microbatches.limits.max_batches must be a positive integer"); + if matches!(get(limits, MAX_BATCHES_KEY), Some(ConfigValue::Bool(_))) { + return Err(invalid()); + } + let value = optional_int(limits, MAX_BATCHES_KEY, 0)?; + if int_below(&value, 1) { + return Err(invalid()); + } + Ok(Some(value)) +} + +/// `_load_microbatches`. +pub(crate) fn load_microbatches(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "microbatches"))?; + validate_allowed_keys(mapping, &["limits"], place(file, "microbatches"))?; + let limits = coerce_mapping(get(mapping, "limits"), place(file, "microbatches.limits"))?; + validate_allowed_keys( + limits, + &[MAX_BATCHES_KEY, "action"], + place(file, "microbatches.limits"), + )?; + let max_batches = positive_max_batches(limits)?; + let action = optional_str(limits, "action")?.unwrap_or_else(|| "error".to_owned()); + if !MICROBATCH_LIMIT_ACTIONS.contains(&action.as_str()) { + return Err(ConfigFailure::plain( + "microbatches.limits.action must be one of: error, warn", + )); + } + Ok(object( + "MicrobatchesConfig", + vec![( + "limits", + object( + "MicrobatchLimitsConfig", + vec![ + ("max_batches", value_or_none(max_batches)), + ("action", member("MicrobatchLimitAction", &action)), + ], + ), + )], + )) +} + +/// `_load_migrations`. +pub(crate) fn load_migrations(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "migrations"))?; + validate_allowed_keys(mapping, &[OLD_NAME_VIEWS_KEY], place(file, "migrations"))?; + let views = match get(mapping, OLD_NAME_VIEWS_KEY) { + None => return Ok(object("MigrationsConfig", Vec::new())), + Some(ConfigValue::Bool(false)) => Built::Value(ConfigValue::Null), + Some(ConfigValue::String(text)) if parse_duration(python_strip(text))?.is_some() => { + Built::Value(ConfigValue::String(python_strip(text).to_owned())) + } + Some(_) => { + return Err(ConfigFailure::at( + file, + "migrations.old_name_views must be a positive duration such as '30d', \ + or false to disable compatibility views at old model names", + )); + } + }; + Ok(object("MigrationsConfig", vec![("old_name_views", views)])) +} + +/// `_load_scopes` and `_load_references`: one boolean policy that defaults to on. +pub(crate) fn load_policy_flag( + payload: Option<&ConfigValue>, + section: (&'static str, &'static str, &'static str), + file: ConfigFile, +) -> Loaded { + let (label, key, class) = section; + let mapping = coerce_mapping(payload, place(file, label))?; + validate_allowed_keys(mapping, &[key], place(file, label))?; + Ok(object( + class, + vec![( + key, + Built::Value(ConfigValue::Bool(optional_bool(mapping, key, true)?)), + )], + )) +} + +fn is_positive_number(value: &ConfigValue) -> bool { + match value { + ConfigValue::Integer(number) => *number > 0, + ConfigValue::BigInteger(text) => !text.starts_with('-'), + ConfigValue::Float(number) => number.is_finite() && *number > ZERO_FLOAT, + _ => false, + } +} + +/// `_load_cost`. +pub(crate) fn load_cost(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "cost"))?; + validate_allowed_keys(mapping, &[USD_PER_CREDIT_KEY], place(file, "cost"))?; + let raw = match get(mapping, USD_PER_CREDIT_KEY) { + None => return Ok(object("CostConfig", Vec::new())), + Some( + raw @ (ConfigValue::Integer(_) | ConfigValue::BigInteger(_) | ConfigValue::Float(_)), + ) => raw, + Some(_) => { + return Err(ConfigFailure::at( + file, + "cost.usd_per_credit must be a number", + )); + } + }; + if !is_positive_number(raw) { + return Err(ConfigFailure::at( + file, + "cost.usd_per_credit must be a finite number greater than zero", + )); + } + Ok(object( + "CostConfig", + vec![ + ("usd_per_credit", Built::Decimal(raw.clone())), + ( + "usd_per_credit_is_default", + Built::Value(ConfigValue::Bool(false)), + ), + ], + )) +} + +/// `_load_constants`. +pub(crate) fn load_constants(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "constants"))?; + validate_allowed_keys(mapping, &["collection_rendering"], place(file, "constants"))?; + let rendering = match get(mapping, "collection_rendering") { + None => COLLECTION_RENDERINGS[0], + Some(ConfigValue::String(text)) => text.as_str(), + Some(_) => { + return Err(ConfigFailure::at( + file, + "constants.collection_rendering must be a string", + )); + } + }; + if !COLLECTION_RENDERINGS.contains(&rendering) { + return Err(ConfigFailure::at( + file, + format!( + "constants.collection_rendering must be one of: {}", + COLLECTION_RENDERINGS.join(", ") + ), + )); + } + Ok(object( + "ConstantsConfig", + vec![( + "collection_rendering", + member("CollectionRendering", rendering), + )], + )) +} + +/// `_load_diff`. +pub(crate) fn load_diff(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "diff"))?; + validate_allowed_keys(mapping, &["query_artifact_ttl"], place(file, "diff"))?; + let ttl = optional_str(mapping, "query_artifact_ttl")? + .unwrap_or_else(|| DEFAULT_QUERY_ARTIFACT_TTL.to_owned()); + if fixed_duration_seconds(&ttl, false)?.is_none() { + return Err(ConfigFailure::at( + file, + "diff.query_artifact_ttl must be a positive fixed duration such as '24h'", + )); + } + Ok(object( + "DiffConfig", + vec![("query_artifact_ttl", Built::Value(ConfigValue::String(ttl)))], + )) +} + +fn janitor_count(mapping: &Entries, key: &str, default: i64) -> Loaded { + optional_int(mapping, key, default) +} + +/// `_load_janitor`. +pub(crate) fn load_janitor(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "janitor"))?; + reject_removed_virtual_keys(mapping, &["max_checkpoints"], place(file, "janitor"))?; + let enabled = optional_bool(mapping, "enabled", false)?; + let retention_days = optional_nullable_int(mapping, "retention_days")?; + let archive = janitor_count(mapping, "archive_retention_days", 14)?; + let history = janitor_count(mapping, "direct_state_history_versions", 20)?; + let delete_tracked_only = optional_bool(mapping, "delete_tracked_only", true)?; + let exclude = string_sequence( + get(mapping, "exclude_patterns"), + place(file, "janitor.exclude_patterns"), + )?; + let checks = [ + (retention_days.as_ref(), "retention_days"), + (Some(&archive), "archive_retention_days"), + (Some(&history), "direct_state_history_versions"), + ]; + if let Some((_, key)) = checks + .iter() + .find(|(value, _)| value.is_some_and(|number| int_below(number, 0))) + { + return Err(ConfigFailure::at( + file, + format!("janitor.{key} must be >= 0"), + )); + } + Ok(object( + "JanitorConfig", + vec![ + ("enabled", Built::Value(ConfigValue::Bool(enabled))), + ("retention_days", value_or_none(retention_days)), + ("archive_retention_days", Built::Value(archive)), + ("direct_state_history_versions", Built::Value(history)), + ( + "delete_tracked_only", + Built::Value(ConfigValue::Bool(delete_tracked_only)), + ), + ("exclude_patterns", Built::Tuple(exclude)), + ], + )) +} + +/// `_load_snapshots`. +pub(crate) fn load_snapshots(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "snapshots"))?; + validate_allowed_keys(mapping, &SNAPSHOT_KEYS, place(file, "snapshots"))?; + let policies: [(&'static str, &[&str], &str); 4] = [ + ( + "current_state_full_refresh", + &SNAPSHOT_FULL_REFRESH_POLICIES, + "deny", + ), + ( + "historical_full_refresh", + &SNAPSHOT_FULL_REFRESH_POLICIES, + "require_confirmation", + ), + ( + "schema_change", + &SNAPSHOT_SCHEMA_CHANGE_POLICIES, + "append_new_columns", + ), + ( + "wildcard_check_schema_change", + &SNAPSHOT_SCHEMA_CHANGE_POLICIES, + "require_confirmation", + ), + ]; + let mut fields: Fields = Vec::new(); + for (key, choices, default) in policies { + let policy = optional_choice(mapping, key, choices, default)?; + fields.push((key, Built::Value(ConfigValue::String(policy)))); + } + Ok(object("SnapshotsConfig", fields)) +} + +fn is_negative(value: Option<&ConfigValue>) -> bool { + value.is_some_and(|number| int_below(number, 0)) +} + +fn is_run_namespace(value: &ConfigValue) -> bool { + matches!(value, ConfigValue::String(text) + if (1..=RUN_NAMESPACE_MAX_LENGTH).contains(&text.len()) + && text.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-'))) +} + +fn local_type_overrides(mapping: &Entries, file: ConfigFile) -> Loaded { + let overrides = coerce_mapping( + get(mapping, "local_type_overrides"), + place(file, "scenario.local_type_overrides"), + )?; + let mut loaded: Vec<(String, Built)> = Vec::new(); + for (dialect, rules) in overrides { + let name = python_strip(key_name(dialect)); + if name.is_empty() { + return Err(ConfigFailure::at( + file, + "scenario.local_type_overrides contains an empty dialect key", + )); + } + loaded = insert_entry(loaded, name.to_owned(), string_mapping(Some(rules), file)?); + } + Ok(Built::Dict(loaded)) +} + +fn snapshot_limits(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "scenario.snapshot_limits"))?; + validate_allowed_keys( + mapping, + &SNAPSHOT_LIMIT_KEYS, + place(file, "scenario.snapshot_limits"), + )?; + let limits = SNAPSHOT_LIMIT_KEYS + .into_iter() + .map(|key| keyed(key, optional_nullable_int(mapping, key))) + .collect::>>()?; + if limits.iter().any(|(_, value)| is_negative(value.as_ref())) { + return Err(ConfigFailure::at( + file, + "scenario.snapshot_limits values must be >= 0", + )); + } + Ok(object( + "ScenarioSnapshotLimitsConfig", + limits + .into_iter() + .map(|(key, value)| (key, value_or_none(value))) + .collect(), + )) +} + +/// `_load_scenario`. +pub(crate) fn load_scenario(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "scenario"))?; + validate_allowed_keys(mapping, &SCENARIO_KEYS, place(file, "scenario"))?; + let overrides = local_type_overrides(mapping, file)?; + let run_namespace = match get(mapping, RUN_NAMESPACE_KEY) { + None => Built::Value(ConfigValue::Null), + Some(value) if is_run_namespace(value) => Built::Value(value.clone()), + Some(_) => { + return Err(ConfigFailure::at( + file, + "scenario.run_namespace must be 1–128 characters: ASCII letters, digits, '-', \ + '_' or '.'", + )); + } + }; + Ok(object( + "ScenarioConfig", + vec![ + ("run_namespace", run_namespace), + ("local_type_overrides", overrides), + ( + "snapshot_limits", + snapshot_limits(get(mapping, "snapshot_limits"), file)?, + ), + ], + )) +} + +fn present_keys<'a>(mapping: &Entries, keys: &[&'a str]) -> Vec<&'a str> { + keys.iter() + .copied() + .filter(|key| has(mapping, key)) + .collect() +} + +/// `_load_dbt`. +pub(crate) fn load_dbt(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "dbt"))?; + let reuse = present_keys(mapping, &DBT_REUSE_KEYS); + if !reuse.is_empty() { + return Err(ConfigFailure::at( + file, + format!( + "[dbt] reuse option(s) were removed: {}; dbt production-ref reuse is no longer supported", + reuse.join(", ") + ), + )); + } + let removed = present_keys(mapping, &DBT_REMOVED_KEYS); + if !removed.is_empty() { + return Err(ConfigFailure::at( + file, + format!( + "[dbt] option(s) were removed: {}; use dbt-native --state/--defer", + removed.join(", ") + ), + )); + } + validate_allowed_keys(mapping, &DBT_KEYS, place(file, "dbt"))?; + let mut fields: Fields = DBT_KEYS[..4] + .iter() + .map(|key| keyed(*key, optional_text_field(mapping, key))) + .collect::>()?; + fields.push(("vars", object_mapping(get(mapping, "vars"), file)?)); + Ok(object("DbtConfig", fields)) +} + +/// `_load_local_dbt`. +pub(crate) fn load_local_dbt(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "dbt"))?; + if has(mapping, DBT_DEFER_CLONE_KEY) { + return Err(ConfigFailure::at( + file, + "[dbt] defer_clone_from was removed; remove it and use dbt-native --state/--defer", + )); + } + validate_allowed_keys(mapping, &LOCAL_DBT_KEYS, place(file, "dbt"))?; + Ok(object( + "LocalDbtConfig", + vec![ + ("target", text_or_none(optional_str(mapping, "target")?)), + ("vars", object_mapping(get(mapping, "vars"), file)?), + ], + )) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/sinks.rs b/crates/sqlbuild-config/src/project/_helpers/sinks.rs new file mode 100644 index 0000000000..2e3765594f --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/sinks.rs @@ -0,0 +1,205 @@ +//! Loader for `[sinks]` and its lifecycle event sinks. + +use crate::models::ConfigValue; +use crate::project::_helpers::fields::{get, key_name, object}; +use crate::project::_helpers::keys::{Place, validate_allowed_keys}; +use crate::project::_helpers::sections::fixed_duration_seconds; +use crate::project::_helpers::targets::seconds_value; +use crate::project::_helpers::values::compare_naturals; +use crate::project::constants::{ + EVENT_EXPORT_KINDS, EVENT_EXPORT_SEVERITIES, EVENT_FILTER_KEYS, EVENT_SINK_KEYS, + SHUTDOWN_TIMEOUT_KEY, SHUTDOWN_TIMEOUT_MAX_SECONDS, +}; +use crate::project::errors::ConfigFailure; +use crate::project::models::{Built, ConfigFile}; +use crate::project::types::{Entries, Loaded}; + +fn event_kinds(mapping: &Entries, place: Place<'_>) -> Loaded { + let Some(value) = get(mapping, "event_kinds") else { + return Ok(Built::Value(ConfigValue::Null)); + }; + let invalid = || { + ConfigFailure::at( + place.file, + format!("{}.event_kinds must be a list of strings", place.label), + ) + }; + let ConfigValue::List(items) = value else { + return Err(invalid()); + }; + let mut kinds: Vec = Vec::new(); + for item in items { + let ConfigValue::String(kind) = item else { + return Err(invalid()); + }; + if !kinds.contains(kind) { + kinds.push(kind.clone()); + } + } + let mut unknown: Vec<&str> = kinds + .iter() + .map(String::as_str) + .filter(|kind| !EVENT_EXPORT_KINDS.contains(kind)) + .collect(); + unknown.sort_unstable(); + if !unknown.is_empty() { + return Err(ConfigFailure::at( + place.file, + format!( + "{}.event_kinds contains unknown kind(s): {}", + place.label, + unknown.join(", ") + ), + )); + } + Ok(Built::FrozenSet(kinds)) +} + +fn min_severity(mapping: &Entries, place: Place<'_>) -> Loaded { + match get(mapping, "min_severity") { + None => Ok(Built::Value(ConfigValue::Null)), + Some(ConfigValue::String(text)) if EVENT_EXPORT_SEVERITIES.contains(&text.as_str()) => { + Ok(Built::Enum("EventExportSeverity", text.clone())) + } + Some(_) => Err(ConfigFailure::at( + place.file, + format!( + "{}.min_severity must be one of: {}", + place.label, + EVENT_EXPORT_SEVERITIES.join(", ") + ), + )), + } +} + +fn load_filter(mapping: &Entries, place: Place<'_>) -> Loaded { + validate_allowed_keys(mapping, &EVENT_FILTER_KEYS, place)?; + Ok(object( + "LifecycleEventSinkFilterConfig", + vec![ + ("event_kinds", event_kinds(mapping, place)?), + ("min_severity", min_severity(mapping, place)?), + ], + )) +} + +fn named_filters(config: &Entries, file: ConfigFile) -> Loaded { + let named = match get(config, "named") { + None => &[][..], + Some(ConfigValue::Map(entries)) => entries.as_slice(), + Some(_) => { + return Err(ConfigFailure::at( + file, + "sinks.lifecycle.named must be a mapping", + )); + } + }; + let mut loaded: Vec<(String, Built)> = Vec::new(); + for (key, value) in named { + let name = key_name(key); + if name.is_empty() { + return Err(ConfigFailure::at( + file, + "sinks.lifecycle.named contains an invalid name", + )); + } + let label = format!("sinks.lifecycle.named.{name}"); + let ConfigValue::Map(entries) = value else { + return Err(ConfigFailure::at( + file, + format!("{label} must be a mapping"), + )); + }; + loaded.push(( + name.to_owned(), + load_filter( + entries, + Place { + file, + label: &label, + }, + )?, + )); + } + Ok(Built::Dict(loaded)) +} + +fn shutdown_timeout(config: &Entries, file: ConfigFile) -> Loaded<(Built, Built)> { + let Some(value) = get(config, SHUTDOWN_TIMEOUT_KEY) else { + return Ok(( + Built::Value(ConfigValue::Null), + Built::Value(ConfigValue::Null), + )); + }; + let invalid = || { + ConfigFailure::at( + file, + format!( + "sinks.lifecycle.{SHUTDOWN_TIMEOUT_KEY} must be a fixed duration from '0s' to \ + '10m' such as '30s'" + ), + ) + }; + let ConfigValue::String(text) = value else { + return Err(invalid()); + }; + let seconds = fixed_duration_seconds(text, true)?.ok_or_else(invalid)?; + let limit = SHUTDOWN_TIMEOUT_MAX_SECONDS.to_string(); + if compare_naturals(&seconds, &limit).is_gt() { + return Err(invalid()); + } + Ok(( + Built::Value(value.clone()), + Built::Value(seconds_value(&seconds)), + )) +} + +fn lifecycle(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let config = match payload { + None => return Ok(object("LifecycleEventSinksConfig", Vec::new())), + Some(ConfigValue::Map(entries)) => entries, + Some(_) => return Err(ConfigFailure::at(file, "sinks.lifecycle must be a mapping")), + }; + let place = Place { + file, + label: "sinks.lifecycle", + }; + validate_allowed_keys(config, &EVENT_SINK_KEYS, place)?; + let named = named_filters(config, file)?; + let defaults: Vec<(ConfigValue, ConfigValue)> = config + .iter() + .filter(|(key, _)| EVENT_FILTER_KEYS.contains(&key_name(key))) + .cloned() + .collect(); + let (timeout, timeout_seconds) = shutdown_timeout(config, file)?; + Ok(object( + "LifecycleEventSinksConfig", + vec![ + ("defaults", load_filter(&defaults, place)?), + ("named", named), + ("shutdown_timeout", timeout), + ("shutdown_timeout_seconds", timeout_seconds), + ], + )) +} + +/// `_load_sinks`. +pub(crate) fn load_sinks(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = match payload { + None => return Ok(object("SinksConfig", Vec::new())), + Some(ConfigValue::Map(entries)) => entries, + Some(_) => return Err(ConfigFailure::at(file, "sinks must be a mapping")), + }; + validate_allowed_keys( + mapping, + &["lifecycle"], + Place { + file, + label: "sinks", + }, + )?; + Ok(object( + "SinksConfig", + vec![("lifecycle", lifecycle(get(mapping, "lifecycle"), file)?)], + )) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs new file mode 100644 index 0000000000..91345408ca --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -0,0 +1,470 @@ +//! Loaders for `[targets]` and `[connections]`. + +use crate::models::ConfigValue; +use crate::project::_helpers::defaults::{ + InheritableField, optional_retention_policy, optional_table_type, +}; +use crate::project::_helpers::fields::{ + bool_or_none, coerce_mapping, get, key_name, keyed, object, optional_nullable_bool, + optional_str, optional_strict_int, optional_text_field, string_mapping, text_or_none, + value_or_none, +}; +use crate::project::_helpers::keys::{ + Place, reject_removed_virtual_keys, reject_unknown_mapping_keys, validate_allowed_keys, +}; +use crate::project::_helpers::sections::fixed_duration_seconds; +use crate::project::_helpers::values::{int_below, is_int}; +use crate::project::constants::{ + CLONE_KEYS, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, MAX_FULL_ROWS_KEY, + RETENTION_MATERIALIZATIONS, TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, +}; +use crate::project::errors::ConfigFailure; +use crate::project::models::{Built, ConfigFile}; +use crate::project::types::{Entries, Fields, Loaded}; +use sqlbuild_core::text::main::python_strip::python_strip; + +/// Policy fields of a target, with the enum each projects to. +const TARGET_POLICIES: [(&str, &str); 3] = [ + ("table_type_downgrade", "TableTypeDowngradePolicy"), + ("time_travel_retention_decrease", "RetentionDecreasePolicy"), + ("missing_migration_origin", "MissingMigrationOriginPolicy"), +]; + +/// One target being loaded: its name and file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct TargetPlace<'a> { + name: &'a str, + file: ConfigFile, +} + +fn none() -> Built { + Built::Value(ConfigValue::Null) +} + +/// `_load_connections`. +pub(crate) fn load_connections(payload: Option<&ConfigValue>, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping( + payload, + Place { + file, + label: "connections", + }, + )?; + let mut loaded: Vec<(String, Built)> = Vec::new(); + for (key, value) in mapping { + let name = key_name(key); + if python_strip(name).is_empty() { + return Err(ConfigFailure::at( + file, + "connections contains an empty name", + )); + } + let label = format!("connections.{name}"); + let entries = coerce_mapping( + Some(value), + Place { + file, + label: &label, + }, + )?; + loaded.push(( + name.to_owned(), + Built::Value(ConfigValue::Map(entries.to_vec())), + )); + } + Ok(Built::Dict(loaded)) +} + +fn execution_limits(payload: Option<&ConfigValue>, label: &str, file: ConfigFile) -> Loaded { + let mapping = coerce_mapping(payload, Place { file, label })?; + validate_allowed_keys(mapping, &EXECUTION_LIMIT_KEYS, Place { file, label })?; + let max_models = optional_strict_int(mapping, "max_models")?; + if max_models + .as_ref() + .is_some_and(|number| int_below(number, 1)) + { + return Err(ConfigFailure::at( + file, + format!("{label}.max_models must be >= 1"), + )); + } + let max_duration = optional_str(mapping, "max_duration")?; + let seconds = match max_duration.as_deref() { + None => none(), + Some(text) => match fixed_duration_seconds(text, false)? { + Some(seconds) => Built::Value(seconds_value(&seconds)), + None => { + return Err(ConfigFailure::at( + file, + format!("{label}.max_duration must be a positive fixed duration such as '30m'"), + )); + } + }, + }; + Ok(object( + "ExecutionLimitsConfig", + vec![ + ("max_models", value_or_none(max_models)), + ("max_duration", text_or_none(max_duration)), + ("max_duration_seconds", seconds), + ("remediation", optional_text_field(mapping, "remediation")?), + ], + )) +} + +/// Whole seconds as a Python integer. +pub(crate) fn seconds_value(seconds: &str) -> ConfigValue { + seconds.parse::().map_or_else( + |_| ConfigValue::BigInteger(seconds.to_owned()), + ConfigValue::Integer, + ) +} + +fn target_connection(mapping: &Entries, target: TargetPlace<'_>) -> Loaded<(Built, Built)> { + match get(mapping, "connection") { + None => Ok((Built::Value(ConfigValue::Map(Vec::new())), none())), + Some(value @ ConfigValue::Map(_)) => Ok((Built::Value(value.clone()), none())), + Some(ConfigValue::String(text)) if !python_strip(text).is_empty() => Ok(( + Built::Value(ConfigValue::Map(Vec::new())), + Built::Value(ConfigValue::String(python_strip(text).to_owned())), + )), + Some(_) => Err(ConfigFailure::at( + target.file, + format!( + "targets.{}.connection must be a non-empty string or mapping", + target.name + ), + )), + } +} + +fn retention_by_materialization(mapping: &Entries, target: TargetPlace<'_>) -> Loaded { + let Some(payload @ ConfigValue::Map(_)) = get(mapping, "time_travel_retention") else { + return Ok(none()); + }; + let label = format!("targets.{}.time_travel_retention", target.name); + let place = Place { + file: target.file, + label: &label, + }; + let retention = coerce_mapping(Some(payload), place)?; + validate_allowed_keys(retention, &RETENTION_MATERIALIZATIONS, place)?; + let mut loaded: Vec<(String, Built)> = Vec::new(); + for materialization in RETENTION_MATERIALIZATIONS { + let field_label = format!("{label}.{materialization}"); + let policy = optional_retention_policy( + retention, + InheritableField { + key: materialization, + label: &field_label, + allow_inherit: false, + }, + target.file, + )?; + if policy != none() { + loaded.push((materialization.to_owned(), policy)); + } + } + Ok(Built::Dict(loaded)) +} + +fn retention_default(mapping: &Entries, target: TargetPlace<'_>) -> Loaded { + if matches!( + get(mapping, "time_travel_retention"), + Some(ConfigValue::Map(_)) + ) { + return Ok(none()); + } + let label = format!("targets.{}.time_travel_retention", target.name); + optional_retention_policy( + mapping, + InheritableField { + key: "time_travel_retention", + label: &label, + allow_inherit: false, + }, + target.file, + ) +} + +fn target_policies(mapping: &Entries, target: TargetPlace<'_>) -> Loaded { + TARGET_POLICIES + .into_iter() + .map(|(key, class)| match get(mapping, key) { + None => Ok((key, none())), + Some(ConfigValue::String(text)) if CONFIRMATION_POLICIES.contains(&text.as_str()) => { + Ok((key, Built::Enum(class, text.clone()))) + } + Some(_) => Err(ConfigFailure::at( + target.file, + format!( + "targets.{}.{key} must be 'deny', 'require_confirmation', or 'allow'", + target.name + ), + )), + }) + .collect() +} + +/// `load_target_diff`. +fn target_diff(payload: Option<&ConfigValue>, target: TargetPlace<'_>) -> Loaded { + let label = format!("targets.{}.diff", target.name); + let entries = match payload { + None => return Ok(object("TargetDiffConfig", Vec::new())), + Some(ConfigValue::Map(entries)) => entries, + Some(_) => { + return Err(ConfigFailure::at( + target.file, + format!("{label} must be a mapping"), + )); + } + }; + let place = Place { + file: target.file, + label: &label, + }; + reject_unknown_mapping_keys(entries, &[MAX_FULL_ROWS_KEY], place)?; + let rows = match get(entries, MAX_FULL_ROWS_KEY) { + None => return Ok(object("TargetDiffConfig", Vec::new())), + Some(ConfigValue::String(text)) if text == UNLIMITED_DIFF_FULL_ROWS => { + Built::Value(ConfigValue::String(text.clone())) + } + Some(value) + if is_int(value) && !matches!(value, ConfigValue::Bool(_)) && !int_below(value, 1) => + { + Built::Value(value.clone()) + } + Some(_) => { + return Err(ConfigFailure::at( + target.file, + format!( + "{label}.{MAX_FULL_ROWS_KEY} must be an integer >= 1 or '{UNLIMITED_DIFF_FULL_ROWS}'" + ), + )); + } + }; + Ok(object("TargetDiffConfig", vec![("max_full_rows", rows)])) +} + +/// `load_target_warehouses`. +fn target_warehouses(payload: Option<&ConfigValue>, target: TargetPlace<'_>) -> Loaded { + let label = format!("targets.{}.warehouses", target.name); + let entries = match payload { + None => return Ok(object("TargetWarehousesConfig", Vec::new())), + Some(ConfigValue::Map(entries)) => entries, + Some(_) => { + return Err(ConfigFailure::at( + target.file, + format!("{label} must be a mapping"), + )); + } + }; + let place = Place { + file: target.file, + label: &label, + }; + reject_unknown_mapping_keys(entries, &WAREHOUSE_GROUPS, place)?; + let fields = WAREHOUSE_GROUPS + .into_iter() + .map(|group| match get(entries, group) { + None => Ok((group, none())), + Some(ConfigValue::String(text)) if !python_strip(text).is_empty() => Ok(( + group, + Built::Value(ConfigValue::String(python_strip(text).to_owned())), + )), + Some(_) => Err(ConfigFailure::at( + target.file, + format!("{label}.{group} must be a non-empty warehouse name"), + )), + }) + .collect::>()?; + Ok(object("TargetWarehousesConfig", fields)) +} + +fn validate_target_keys(mapping: &Entries, target: TargetPlace<'_>) -> Result<(), ConfigFailure> { + let label = format!("targets.{}", target.name); + let place = Place { + file: target.file, + label: &label, + }; + reject_removed_virtual_keys(mapping, &["state", "changes_only"], place)?; + validate_allowed_keys(mapping, &TARGET_KEYS, place) +} + +fn clone_mapping<'a>(mapping: &'a Entries, target: TargetPlace<'_>) -> Loaded<&'a Entries> { + let label = format!("targets.{}.clone", target.name); + let place = Place { + file: target.file, + label: &label, + }; + let clone = coerce_mapping(get(mapping, "clone"), place)?; + validate_allowed_keys(clone, &CLONE_KEYS, place)?; + Ok(clone) +} + +fn text_fields(mapping: &Entries) -> Loaded { + [ + "database", + "schema", + "loader_schema", + "defer_sources_to", + "defer_clone_from", + ] + .into_iter() + .map(|key| keyed(key, optional_text_field(mapping, key))) + .collect() +} + +fn retention_fields(mapping: &Entries, target: TargetPlace<'_>) -> Loaded { + let table_type_label = format!("targets.{}.default_table_type", target.name); + let mut fields: Fields = vec![ + ( + "compile_cache", + bool_or_none(optional_nullable_bool(mapping, "compile_cache")?), + ), + ("time_travel_retention", retention_default(mapping, target)?), + ( + "time_travel_retention_by_materialization", + retention_by_materialization(mapping, target)?, + ), + ( + "owns_time_travel_retention_namespace", + bool_or_none(optional_nullable_bool( + mapping, + "owns_time_travel_retention_namespace", + )?), + ), + ( + "default_table_type", + optional_table_type( + mapping, + InheritableField { + key: "default_table_type", + label: &table_type_label, + allow_inherit: false, + }, + target.file, + )?, + ), + ]; + fields.extend(target_policies(mapping, target)?); + Ok(fields) +} + +/// One `LocalTargetConfig`'s fields, in the Python loader's evaluation order. +fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded { + let label = format!("targets.{}", target.name); + let mapping = coerce_mapping( + Some(payload), + Place { + file: target.file, + label: &label, + }, + )?; + validate_target_keys(mapping, target)?; + let clone = clone_mapping(mapping, target)?; + let limits_label = format!("targets.{}.execution_limits", target.name); + let limits = execution_limits(get(mapping, "execution_limits"), &limits_label, target.file)?; + let (connection, connection_name) = target_connection(mapping, target)?; + let mut fields: Fields = vec![ + ("connection", connection), + ("connection_name", connection_name), + ("vars", string_mapping(get(mapping, "vars"), target.file)?), + ]; + fields.extend(text_fields(mapping)?); + fields.extend(retention_fields(mapping, target)?); + fields.push(("execution_limits", limits)); + fields.push(("diff", target_diff(get(mapping, "diff"), target)?)); + fields.push(( + "warehouses", + target_warehouses(get(mapping, "warehouses"), target)?, + )); + let clone_fields: Fields = CLONE_KEYS + .into_iter() + .map(|key| keyed(key, optional_nullable_bool(clone, key).map(bool_or_none))) + .collect::>()?; + fields.push(("clone", object("LocalClonePolicy", clone_fields))); + Ok(fields) +} + +/// `_load_local_targets`: each target's fields by name. +pub(crate) fn load_target_fields( + payload: Option<&ConfigValue>, + file: ConfigFile, +) -> Loaded> { + let mapping = coerce_mapping( + payload, + Place { + file, + label: "targets", + }, + )?; + mapping + .iter() + .map(|(key, value)| { + let name = key_name(key); + keyed( + name.to_owned(), + local_target(value, TargetPlace { name, file }), + ) + }) + .collect() +} + +fn project_field(key: &'static str, value: Built) -> Option<(&'static str, Built)> { + let is_none = value == none(); + match key { + "clone" => Some((key, project_clone(value))), + "time_travel_retention_by_materialization" if is_none => { + Some((key, Built::Dict(Vec::new()))) + } + "owns_time_travel_retention_namespace" => Some(( + key, + Built::Value(ConfigValue::Bool( + value == Built::Value(ConfigValue::Bool(true)), + )), + )), + "table_type_downgrade" | "time_travel_retention_decrease" | "missing_migration_origin" + if is_none => + { + None + } + _ => Some((key, value)), + } +} + +fn project_clone(value: Built) -> Built { + let Built::Object(_, fields) = value else { + return value; + }; + object( + "ClonePolicy", + fields + .into_iter() + .map(|(key, flag)| { + ( + key, + Built::Value(ConfigValue::Bool( + flag == Built::Value(ConfigValue::Bool(true)), + )), + ) + }) + .collect(), + ) +} + +/// `_project_target`: a shared target, with the local-only `None`s resolved to defaults. +pub(crate) fn project_target(fields: Fields) -> Built { + object( + "TargetConfig", + fields + .into_iter() + .filter_map(|(key, value)| project_field(key, value)) + .collect(), + ) +} + +/// A local target. +pub(crate) fn local_target_config(fields: Fields) -> Built { + object("LocalTargetConfig", fields) +} diff --git a/crates/sqlbuild-config/src/project/_helpers/values.rs b/crates/sqlbuild-config/src/project/_helpers/values.rs new file mode 100644 index 0000000000..feb3f8d8d4 --- /dev/null +++ b/crates/sqlbuild-config/src/project/_helpers/values.rs @@ -0,0 +1,152 @@ +//! Python's `isinstance`, truth, ordering and equality over loaded values. + +use crate::constants::FEBRUARY; +use crate::models::{ConfigDateTime, ConfigValue}; +use crate::project::constants::ZERO_FLOAT; +use std::cmp::Ordering; + +const SECONDS_PER_DAY: i64 = 86_400; +const SECONDS_PER_HOUR: i64 = 3_600; +const SECONDS_PER_MINUTE: i64 = 60; +const MICROSECONDS_PER_SECOND: i64 = 1_000_000; + +/// `isinstance(value, int)`, which includes booleans. +pub(crate) fn is_int(value: &ConfigValue) -> bool { + matches!( + value, + ConfigValue::Integer(_) | ConfigValue::BigInteger(_) | ConfigValue::Bool(_) + ) +} + +/// `value < bound` for a Python integer `value`. +pub(crate) fn int_below(value: &ConfigValue, bound: i64) -> bool { + match value { + ConfigValue::Integer(number) => *number < bound, + ConfigValue::Bool(flag) => i64::from(*flag) < bound, + ConfigValue::BigInteger(text) => text.starts_with('-'), + _ => false, + } +} + +/// `bool(value)`. +pub(crate) fn is_truthy(value: &ConfigValue) -> bool { + match value { + ConfigValue::Null => false, + ConfigValue::Bool(flag) => *flag, + ConfigValue::Integer(number) => *number != 0, + ConfigValue::Float(number) => *number != ZERO_FLOAT, + ConfigValue::String(text) => !text.is_empty(), + ConfigValue::List(items) => !items.is_empty(), + ConfigValue::Map(entries) => !entries.is_empty(), + ConfigValue::BigInteger(_) + | ConfigValue::Date(_) + | ConfigValue::DateTime(_) + | ConfigValue::Time(_) => true, + } +} + +/// The exact decimal digits of an integral number, signed, or `None` for any other value. +fn integral_text(value: &ConfigValue) -> Option { + match value { + ConfigValue::Bool(flag) => Some(i64::from(*flag).to_string()), + ConfigValue::Integer(number) => Some(number.to_string()), + ConfigValue::BigInteger(text) => Some(text.clone()), + ConfigValue::Float(number) if *number == ZERO_FLOAT => Some("0".to_owned()), + ConfigValue::Float(number) if number.is_finite() && number.fract() == ZERO_FLOAT => { + Some(format!("{number:.0}")) + } + _ => None, + } +} + +fn is_number(value: &ConfigValue) -> bool { + is_int(value) || matches!(value, ConfigValue::Float(_)) +} + +/// `left == right` between Python numbers, comparing exactly as Python does. +fn numbers_equal(left: &ConfigValue, right: &ConfigValue) -> bool { + match (left, right) { + (ConfigValue::Float(first), ConfigValue::Float(second)) => first == second, + _ => integral_text(left).is_some() && integral_text(left) == integral_text(right), + } +} + +/// Microseconds since the epoch-free origin of an aware datetime, in UTC. +fn utc_microseconds(moment: &ConfigDateTime, offset: i32) -> i128 { + let days = days_from_civil( + i64::from(moment.date.year), + i64::from(moment.date.month), + i64::from(moment.date.day), + ); + let seconds = days * SECONDS_PER_DAY + + i64::from(moment.time.hour) * SECONDS_PER_HOUR + + i64::from(moment.time.minute) * SECONDS_PER_MINUTE + + i64::from(moment.time.second) + - i64::from(offset); + i128::from(seconds) * i128::from(MICROSECONDS_PER_SECOND) + i128::from(moment.time.microsecond) +} + +/// Days from 0000-03-01 to a proleptic Gregorian date. +fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { + let shifted = if month <= FEBRUARY { year - 1 } else { year }; + let era = shifted.div_euclid(400); + let year_of_era = shifted - era * 400; + let month_index = (month + 9) % 12; + let day_of_year = (153 * month_index + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era +} + +fn datetimes_equal(left: &ConfigDateTime, right: &ConfigDateTime) -> bool { + match (left.utc_offset_seconds, right.utc_offset_seconds) { + (None, None) => left == right, + (Some(first), Some(second)) => { + utc_microseconds(left, first) == utc_microseconds(right, second) + } + _ => false, + } +} + +fn entry_equal(entry: &(ConfigValue, ConfigValue), right: &[(ConfigValue, ConfigValue)]) -> bool { + let (key, value) = entry; + for (candidate, other) in right { + if python_equal(candidate, key) { + return python_equal(value, other); + } + } + false +} + +fn maps_equal(left: &[(ConfigValue, ConfigValue)], right: &[(ConfigValue, ConfigValue)]) -> bool { + left.len() == right.len() && left.iter().all(|entry| entry_equal(entry, right)) +} + +/// `left == right` with Python's semantics for the loaded value domain. +pub(crate) fn python_equal(left: &ConfigValue, right: &ConfigValue) -> bool { + if is_number(left) && is_number(right) { + return numbers_equal(left, right); + } + match (left, right) { + (ConfigValue::Null, ConfigValue::Null) => true, + (ConfigValue::String(first), ConfigValue::String(second)) => first == second, + (ConfigValue::Date(first), ConfigValue::Date(second)) => first == second, + (ConfigValue::Time(first), ConfigValue::Time(second)) => first == second, + (ConfigValue::DateTime(first), ConfigValue::DateTime(second)) => { + datetimes_equal(first, second) + } + (ConfigValue::List(first), ConfigValue::List(second)) => { + first.len() == second.len() + && first + .iter() + .zip(second) + .all(|(item, other)| python_equal(item, other)) + } + (ConfigValue::Map(first), ConfigValue::Map(second)) => maps_equal(first, second), + _ => false, + } +} + +/// Compare two non-negative decimal digit strings without leading zeros. +pub(crate) fn compare_naturals(left: &str, right: &str) -> Ordering { + left.len().cmp(&right.len()).then_with(|| left.cmp(right)) +} diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 850458ae50..3e4db042c2 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -1,9 +1,248 @@ -//! Project configuration file names and keys. +//! Project configuration file names, keys and the vocabularies their values come from. pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; -pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml"; -pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml"; +pub(crate) const SETTINGS_SECTION: &str = "settings"; pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; pub(crate) const LEGACY_SQL_VALIDATION_KEY: &str = "sql_validation"; pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis"; +pub(crate) const QUERY_CHANGE_TRACKING_KEY: &str = "query_change_tracking"; +pub(crate) const CONCURRENCY_KEY: &str = "concurrency"; +pub(crate) const LEGACY_CONCURRENCY_KEY: &str = "max_concurrency"; +pub(crate) const MAX_BATCHES_KEY: &str = "max_batches"; +pub(crate) const BATCH_CONCURRENCY_KEY: &str = "batch_concurrency"; +pub(crate) const MAX_MICROBATCHES_KEY: &str = "max_microbatches"; +pub(crate) const OLD_NAME_VIEWS_KEY: &str = "old_name_views"; +pub(crate) const USD_PER_CREDIT_KEY: &str = "usd_per_credit"; +pub(crate) const RUN_NAMESPACE_KEY: &str = "run_namespace"; +pub(crate) const SHUTDOWN_TIMEOUT_KEY: &str = "shutdown_timeout"; +pub(crate) const LEGACY_EVENT_EXPORTERS_KEY: &str = "event_exporters"; +pub(crate) const MAX_FULL_ROWS_KEY: &str = "max_full_rows"; +pub(crate) const DBT_DEFER_CLONE_KEY: &str = "defer_clone_from"; +pub(crate) const DBT_REUSE_KEYS: [&str; 2] = ["reuse_from", "production_ref"]; +pub(crate) const DBT_REMOVED_KEYS: [&str; 2] = ["defer_clone_from", "replay_on_change"]; +pub(crate) const ON_SCHEMA_CHANGE_KEY: &str = "on_schema_change"; +pub(crate) const REPLAY_ON_CHANGE_KEY: &str = "replay_on_change"; +pub(crate) const CHANGE_POLICY_KEYS: [&str; 2] = [ON_SCHEMA_CHANGE_KEY, REPLAY_ON_CHANGE_KEY]; +pub(crate) const TEMPLATE_OPEN_TOKEN: &str = "${"; +pub(crate) const MODELS_DIRECTORY_NAME: &str = "models"; +pub(crate) const UNLIMITED_DIFF_FULL_ROWS: &str = "unlimited"; +pub(crate) const ZERO_DAY_DURATION: &str = "0d"; +pub(crate) const ZERO_FIXED_DURATION: &str = "0s"; +pub(crate) const DEFAULT_QUERY_ARTIFACT_TTL: &str = "24h"; +pub(crate) const BOUNDED_REPLAY_PREFIX: &str = "bounded-"; +pub(crate) const ZERO_FLOAT: f64 = 0.0; +pub(crate) const SHUTDOWN_TIMEOUT_MAX_SECONDS: u64 = 600; +pub(crate) const SUGGESTION_CUTOFF: f64 = 0.6; +pub(crate) const LISTED_SUPPORTED_KEYS_LIMIT: usize = 12; +pub(crate) const RUN_NAMESPACE_MAX_LENGTH: usize = 128; +pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; +pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; + +/// The `SettingsConfig` fields, which `[settings]` may set. +pub(crate) const SETTINGS_KEYS: [&str; 11] = [ + "sql_analysis", + "require_sql_analysis", + "query_change_tracking", + "column_contract_mode", + "concurrency", + "auto_load_sources", + "microbatch_concurrency", + "microbatch_unaccounted_partition_policy", + "table_promotion_mode", + "default_audit_severity", + "default_audit_run_scope", +]; +pub(crate) const DEFAULTS_KEYS: [&str; 32] = [ + "append_cursor_inclusive", + "batch_concurrency", + "batch_size", + "contract", + "cursor_end", + "cursor_future_action", + "cursor_future_max_distance", + "cursor_start", + "cursor_start_max_action", + "cursor_start_max_ahead", + "cursor_watermark_mode", + "database", + "full_refresh", + "function_database", + "function_schema", + "incremental_mode", + "incremental_strategy", + "lookback", + "materialized", + "max_microbatches", + "merge_exclude_columns", + "microbatch_strategy", + "replay_on_change", + "row_diff_exclude_columns", + "row_diff_sample_rows", + "row_diff_sample_seed", + "row_diff_tolerances", + "schema", + "seed_database", + "seed_schema", + "tags", + "unaccounted_partition_policy", +]; +/// The MODEL header keys, which `[path_defaults]` entries may set. +pub(crate) const MODEL_HEADER_KEYS: [&str; 67] = [ + "alias", + "append_cursor_inclusive", + "audit_factories", + "audits", + "batch_concurrency", + "batch_size", + "check_columns", + "columns", + "config", + "constants", + "contract", + "cursor", + "cursor_end", + "cursor_future_action", + "cursor_future_max_distance", + "cursor_grain", + "cursor_inputs", + "cursor_start", + "cursor_start_max_action", + "cursor_start_max_ahead", + "cursor_type", + "cursor_watermark_mode", + "database", + "description", + "dynamic_columns", + "enabled", + "enums", + "full_refresh", + "historical_input", + "incremental_mode", + "incremental_strategy", + "initial_valid_from", + "invalidate_hard_deletes", + "lookback", + "materialized", + "max_microbatches", + "merge_exclude_columns", + "microbatch_limit", + "microbatch_strategy", + "migrate_force", + "migrate_from", + "model_schema", + "observed_at", + "old_name_view", + "on_schema_change", + "placeholders", + "post_hooks", + "pre_hooks", + "replay_on_change", + "row_diff_exclude_columns", + "row_diff_sample_rows", + "row_diff_sample_seed", + "row_diff_tolerances", + "schema", + "snapshot_full_refresh", + "snapshot_schema_change", + "snapshot_strategy", + "sql_analysis", + "sql_validation", + "table_type", + "tags", + "time_travel_retention", + "unaccounted_partition_policy", + "unique_key", + "updated_at", + "valid_from_column", + "valid_to_column", +]; +pub(crate) const TARGET_KEYS: [&str; 18] = [ + "connection", + "vars", + "database", + "schema", + "loader_schema", + "defer_sources_to", + "defer_clone_from", + "compile_cache", + "time_travel_retention", + "owns_time_travel_retention_namespace", + "default_table_type", + "table_type_downgrade", + "time_travel_retention_decrease", + "missing_migration_origin", + "clone", + "execution_limits", + "diff", + "warehouses", +]; +pub(crate) const CLONE_KEYS: [&str; 2] = ["allow_as_clone_origin", "allow_as_clone_destination"]; +pub(crate) const EXECUTION_LIMIT_KEYS: [&str; 3] = ["max_models", "max_duration", "remediation"]; +pub(crate) const SNAPSHOT_KEYS: [&str; 4] = [ + "current_state_full_refresh", + "historical_full_refresh", + "schema_change", + "wildcard_check_schema_change", +]; +pub(crate) const SNAPSHOT_LIMIT_KEYS: [&str; 4] = [ + "max_rows_per_relation", + "max_total_rows", + "max_bytes_per_relation", + "max_total_bytes", +]; +pub(crate) const SCENARIO_KEYS: [&str; 3] = + ["local_type_overrides", "snapshot_limits", RUN_NAMESPACE_KEY]; +pub(crate) const DBT_KEYS: [&str; 5] = [ + "project_dir", + "profiles_dir", + "target", + "target_path", + "vars", +]; +pub(crate) const LOCAL_DBT_KEYS: [&str; 2] = ["target", "vars"]; +pub(crate) const EVENT_FILTER_KEYS: [&str; 2] = ["event_kinds", "min_severity"]; +pub(crate) const EVENT_SINK_KEYS: [&str; 4] = + ["event_kinds", "min_severity", "named", SHUTDOWN_TIMEOUT_KEY]; +pub(crate) const EVENT_EXPORT_KINDS: [&str; 8] = [ + "audit", + "invocation", + "microbatch", + "operation", + "resource", + "retry", + "run", + "statement", +]; +/// `EVENT_EXPORT_SEVERITIES` in its declared order, which error messages list. +pub(crate) const EVENT_EXPORT_SEVERITIES: [&str; 4] = ["debug", "info", "warning", "error"]; +/// `TIME_TRAVEL_RETENTION_MATERIALIZATIONS` in declared order. +pub(crate) const RETENTION_MATERIALIZATIONS: [&str; 3] = ["table", "incremental", "snapshot"]; +pub(crate) const MATERIALIZATION_DEFAULT_KEYS: [&str; 2] = ["time_travel_retention", "table_type"]; +pub(crate) const WAREHOUSE_GROUPS: [&str; 2] = ["build", "query"]; +pub(crate) const SNAPSHOT_FULL_REFRESH_POLICIES: [&str; 3] = + ["allow", "deny", "require_confirmation"]; +pub(crate) const SNAPSHOT_SCHEMA_CHANGE_POLICIES: [&str; 3] = + ["append_new_columns", "deny", "require_confirmation"]; +pub(crate) const UNACCOUNTED_PARTITION_POLICIES: [&str; 3] = + ["recover_all", "recover_empty", "synthesize"]; +pub(crate) const COLUMN_CONTRACT_MODES: [&str; 2] = ["implicit", "explicit"]; +pub(crate) const AUDIT_SEVERITIES: [&str; 2] = ["warn", "error"]; +pub(crate) const COLLECTION_RENDERINGS: [&str; 2] = ["value_list", "array"]; +pub(crate) const FUTURE_CURSOR_ACTIONS: [&str; 2] = ["cap", "error"]; +pub(crate) const MICROBATCH_LIMIT_ACTIONS: [&str; 2] = ["error", "warn"]; +pub(crate) const CONTRACT_POLICIES: [&str; 2] = ["none", "enforced"]; +pub(crate) const TABLE_TYPES: [&str; 2] = ["permanent", "transient"]; +pub(crate) const CONFIRMATION_POLICIES: [&str; 3] = ["deny", "require_confirmation", "allow"]; +pub(crate) const INHERIT_VALUE: &str = "inherit"; +pub(crate) const DISABLED_VALUE: &str = "disabled"; +/// `OnSchemaChange` values, sorted as the error lists them. +pub(crate) const ON_SCHEMA_CHANGE_VALUES: [&str; 4] = + ["append_new_columns", "fail", "ignore", "sync_all_columns"]; +pub(crate) const REPLAY_ON_CHANGE_VALUES: [&str; 3] = ["forward", "full", "bounded-"]; +pub(crate) const ON_SCHEMA_CHANGE_EXAMPLE: &str = "append_new_columns"; +pub(crate) const REPLAY_ON_CHANGE_EXAMPLE: &str = "bounded-14d"; +pub(crate) const GLOB_SEGMENTS: [&str; 2] = ["*", "**"]; +pub(crate) const UNSUPPORTED_GLOB_MARKERS: [char; 3] = ['*', '?', '[']; +/// CPython's default `sys.get_int_max_str_digits()`, beyond which `int()` raises. +pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4_300; diff --git a/crates/sqlbuild-config/src/project/errors.rs b/crates/sqlbuild-config/src/project/errors.rs new file mode 100644 index 0000000000..0db0adb024 --- /dev/null +++ b/crates/sqlbuild-config/src/project/errors.rs @@ -0,0 +1,110 @@ +//! Why a project configuration file is rejected. + +use crate::project::constants::PYTHON_INT_MAX_STR_DIGITS; +use crate::project::models::ConfigFile; + +/// One piece of a rejection message: fixed text or the path of a configuration file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum MessagePiece { + Text(String), + File(ConfigFile), +} + +/// The Python exception a rejection raises. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FailureClass { + /// `ProjectConfigError`. + #[default] + ProjectConfig, + /// The `ValueError` `int()` raises for more digits than CPython converts. + IntegerDigits, +} + +/// A rejection message, its optional help, and the exception it raises. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConfigFailure { + pub message: Vec, + pub help: Option, + pub class: FailureClass, +} + +impl ConfigFailure { + /// A message without a file path. + pub fn plain(text: impl Into) -> Self { + Self { + message: vec![MessagePiece::Text(text.into())], + help: None, + class: FailureClass::ProjectConfig, + } + } + + /// A message that starts with the path of `file`. + pub fn at(file: ConfigFile, text: impl AsRef) -> Self { + Self { + message: vec![ + MessagePiece::File(file), + MessagePiece::Text(format!(" {}", text.as_ref())), + ], + help: None, + class: FailureClass::ProjectConfig, + } + } + + /// The `ValueError` of `int()` for a decimal string of `digits` digits. + pub fn integer_digits(digits: usize) -> Self { + Self { + class: FailureClass::IntegerDigits, + ..Self::plain(format!( + "Exceeds the limit ({PYTHON_INT_MAX_STR_DIGITS} digits) for integer string \ + conversion: value has {digits} digits; use sys.set_int_max_str_digits() to \ + increase the limit" + )) + } + } + + /// This failure with help text. + pub fn with_help(self, help: impl Into) -> Self { + Self { + help: Some(help.into()), + ..self + } + } + + /// The message with each file path rendered by `path`. + pub fn render(&self, path: impl Fn(ConfigFile) -> String) -> String { + self.message + .iter() + .map(|piece| match piece { + MessagePiece::Text(text) => text.clone(), + MessagePiece::File(file) => path(*file), + }) + .collect() + } +} + +/// Why a configuration file could not be loaded. +#[derive(Debug)] +pub enum LoadFailure { + /// The file is rejected with a `ProjectConfigError`. + Config(ConfigFailure), + /// The file could not be read. + Io(ConfigFile, std::io::Error), + /// The file is not UTF-8; Python's decoder raises for these bytes. + Utf8(Vec), +} + +impl LoadFailure { + /// The `ProjectConfigError` this failure raises, if it is one. + pub fn into_config_failure(self) -> Option { + match self { + Self::Config(failure) => Some(failure), + Self::Io(..) | Self::Utf8(_) => None, + } + } +} + +impl From for LoadFailure { + fn from(failure: ConfigFailure) -> Self { + Self::Config(failure) + } +} diff --git a/crates/sqlbuild-config/src/project/main/load_local_config.rs b/crates/sqlbuild-config/src/project/main/load_local_config.rs new file mode 100644 index 0000000000..a1612d95df --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/load_local_config.rs @@ -0,0 +1,9 @@ +//! Load and validate `sqlbuild_local.toml`. + +use crate::project::errors::LoadFailure; +use crate::project::models::{Built, ConfigPaths}; + +/// `load_local_config`: the validated `LocalConfig`, or the default when the file is absent. +pub fn load_local_config(paths: &ConfigPaths) -> Result { + crate::project::_helpers::documents::local_config(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/load_project_config.rs b/crates/sqlbuild-config/src/project/main/load_project_config.rs new file mode 100644 index 0000000000..97e6856413 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/load_project_config.rs @@ -0,0 +1,9 @@ +//! Load and validate `sqlbuild_project.toml`. + +use crate::project::errors::LoadFailure; +use crate::project::models::{Built, ConfigPaths}; + +/// `load_project_config`: the validated `ProjectConfig` of `sqlbuild_project.toml`. +pub fn load_project_config(paths: &ConfigPaths) -> Result { + crate::project::_helpers::documents::project_config(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/local_sql_analysis_failure.rs b/crates/sqlbuild-config/src/project/main/local_sql_analysis_failure.rs new file mode 100644 index 0000000000..ab415ef4b3 --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/local_sql_analysis_failure.rs @@ -0,0 +1,12 @@ +//! Reject a local project-wide SQL analysis opt-out the project forbids. + +use crate::project::errors::ConfigFailure; +use crate::project::models::ConfigPaths; + +/// `validate_local_sql_analysis_policy` over (required, overridden, enabled) SQL analysis flags. +pub fn local_sql_analysis_failure( + paths: &ConfigPaths, + flags: (bool, bool, bool), +) -> Option { + crate::project::_helpers::documents::local_sql_analysis_failure(paths, flags) +} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 846d5991cb..132ce189b6 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -1,2 +1,4 @@ -pub mod read_local_config; -pub mod read_project_config; +pub mod load_local_config; +pub mod load_project_config; +pub mod local_sql_analysis_failure; +pub mod resolve_config_file; diff --git a/crates/sqlbuild-config/src/project/main/read_local_config.rs b/crates/sqlbuild-config/src/project/main/read_local_config.rs deleted file mode 100644 index 70d30f2e34..0000000000 --- a/crates/sqlbuild-config/src/project/main/read_local_config.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Read the discovery fields of `sqlbuild_local.toml`. - -use crate::errors::ConfigError; -use crate::project::_helpers::fields::{ - flag, lookup, optional_text, section, section_keys, text_mapping, -}; -use crate::project::_helpers::files::{load_toml_file, reject_legacy}; -use crate::project::constants::{ - LEGACY_LOCAL_CONFIG_FILENAME, LEGACY_SQL_VALIDATION_KEY, LOCAL_CONFIG_FILENAME, - SQL_ANALYSIS_KEY, -}; -use crate::project::models::LocalConfigFile; -use std::path::Path; - -/// Read `sqlbuild_local.toml` in `project_dir`, or the defaults when it does not exist. -pub fn read_local_config(project_dir: &Path) -> Result { - reject_legacy( - project_dir, - LOCAL_CONFIG_FILENAME, - LEGACY_LOCAL_CONFIG_FILENAME, - )?; - let path = project_dir.join(LOCAL_CONFIG_FILENAME); - if !path.exists() { - return Ok(LocalConfigFile::default()); - } - let table = load_toml_file(&path)?; - let settings = section(&table, "settings")?; - let analysis_key = [SQL_ANALYSIS_KEY, LEGACY_SQL_VALIDATION_KEY] - .into_iter() - .find(|key| lookup(settings, key).is_some()); - Ok(LocalConfigFile { - target: optional_text(&table, "target")?, - adapter: optional_text(&table, "adapter")?, - sql_analysis: analysis_key - .map(|key| flag(settings, key, true)) - .transpose()?, - vars: text_mapping(&table, "vars")?, - target_names: section_keys(&table, "targets")?, - }) -} diff --git a/crates/sqlbuild-config/src/project/main/read_project_config.rs b/crates/sqlbuild-config/src/project/main/read_project_config.rs deleted file mode 100644 index 1c345a5b16..0000000000 --- a/crates/sqlbuild-config/src/project/main/read_project_config.rs +++ /dev/null @@ -1,76 +0,0 @@ -//! Read the discovery fields of `sqlbuild_project.toml`. - -use crate::errors::{ConfigError, ConfigErrorKind}; -use crate::models::ConfigValue; -use crate::project::_helpers::fields::{ - flag, key_text, lookup, optional_text, required_text, section, section_keys, text_mapping, -}; -use crate::project::_helpers::files::{load_toml_file, reject_legacy}; -use crate::project::constants::{ - LEGACY_PROJECT_CONFIG_FILENAME, LEGACY_SQL_VALIDATION_KEY, PROJECT_CONFIG_FILENAME, - REQUIRE_SQL_ANALYSIS_KEY, SQL_ANALYSIS_KEY, -}; -use crate::project::models::{ProjectConfigFile, ProjectSettings}; -use std::path::Path; - -fn settings(table: &ConfigValue) -> Result { - let entries = section(table, "settings")?; - let current = lookup(entries, SQL_ANALYSIS_KEY); - let legacy = lookup(entries, LEGACY_SQL_VALIDATION_KEY); - if current.is_some() && legacy.is_some() && current != legacy { - return Err(ConfigError::new( - ConfigErrorKind::InvalidField, - "settings.sql_analysis conflicts with legacy settings.sql_validation", - )); - } - let analysis_key = if current.is_some() { - SQL_ANALYSIS_KEY - } else { - LEGACY_SQL_VALIDATION_KEY - }; - Ok(ProjectSettings { - sql_analysis: flag(entries, analysis_key, true)?, - require_sql_analysis: flag(entries, REQUIRE_SQL_ANALYSIS_KEY, false)?, - }) -} - -fn path_defaults(table: &ConfigValue) -> Result, ConfigError> { - section(table, "path_defaults")? - .iter() - .map(|(key, value)| { - let name = key_text(key)?; - match value { - ConfigValue::Map(_) => Ok((name, value.clone())), - _ => Err(ConfigError::new( - ConfigErrorKind::InvalidField, - format!("path_defaults['{name}'] must be a mapping"), - )), - } - }) - .collect() -} - -/// Read `sqlbuild_project.toml` in `project_dir`; Python's loader remains the full validator. -pub fn read_project_config(project_dir: &Path) -> Result { - reject_legacy( - project_dir, - PROJECT_CONFIG_FILENAME, - LEGACY_PROJECT_CONFIG_FILENAME, - )?; - let table = load_toml_file(&project_dir.join(PROJECT_CONFIG_FILENAME))?; - Ok(ProjectConfigFile { - name: required_text(&table, "name")?, - adapter: required_text(&table, "adapter")?, - default_target: optional_text(&table, "default_target")?, - settings: settings(&table)?, - enforce_placement: flag(section(&table, "scopes")?, "enforce_placement", true)?, - enforce_explicit_references: flag( - section(&table, "references")?, - "enforce_explicit", - true, - )?, - vars: text_mapping(&table, "vars")?, - path_defaults: path_defaults(&table)?, - target_names: section_keys(&table, "targets")?, - }) -} diff --git a/crates/sqlbuild-config/src/project/main/resolve_config_file.rs b/crates/sqlbuild-config/src/project/main/resolve_config_file.rs new file mode 100644 index 0000000000..43686ce63d --- /dev/null +++ b/crates/sqlbuild-config/src/project/main/resolve_config_file.rs @@ -0,0 +1,9 @@ +//! Which configuration file a loader reads. + +use crate::project::errors::ConfigFailure; +use crate::project::models::{ConfigFile, ConfigPaths}; + +/// The project or local TOML file, rejecting a legacy YAML file that has no TOML replacement. +pub fn resolve_config_file(paths: &ConfigPaths, local: bool) -> Result { + crate::project::_helpers::documents::resolve(paths, local) +} diff --git a/crates/sqlbuild-config/src/project/mod.rs b/crates/sqlbuild-config/src/project/mod.rs index 1e64f3d116..38a9736573 100644 --- a/crates/sqlbuild-config/src/project/mod.rs +++ b/crates/sqlbuild-config/src/project/mod.rs @@ -1,7 +1,9 @@ pub(crate) mod _helpers; pub(crate) mod constants; +pub mod errors; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index 5cdcb3fff1..2874ba17bb 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -1,34 +1,40 @@ -//! The project configuration fields native discovery reads. +//! The loaded project configuration, described as the Python objects it projects to. use crate::models::ConfigValue; -/// Shared settings discovery reads from `[settings]`. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ProjectSettings { - pub sql_analysis: bool, - pub require_sql_analysis: bool, +/// One configuration file a loader reads or names in a message. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConfigFile { + Project, + LegacyProject, + Local, + LegacyLocal, } -/// The discovery fields of `sqlbuild_project.toml`, with Python's defaults and stripping. -#[derive(Clone, Debug, PartialEq)] -pub struct ProjectConfigFile { - pub name: String, - pub adapter: String, - pub default_target: Option, - pub settings: ProjectSettings, - pub enforce_placement: bool, - pub enforce_explicit_references: bool, - pub vars: Vec<(String, String)>, - pub path_defaults: Vec<(String, ConfigValue)>, - pub target_names: Vec, +/// The paths of the project and local configuration files and their legacy YAML names. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConfigPaths { + pub project: std::path::PathBuf, + pub legacy_project: std::path::PathBuf, + pub local: std::path::PathBuf, + pub legacy_local: std::path::PathBuf, } -/// The discovery fields of `sqlbuild_local.toml`; an absent file reads as the default. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct LocalConfigFile { - pub target: Option, - pub adapter: Option, - pub sql_analysis: Option, - pub vars: Vec<(String, String)>, - pub target_names: Vec, +/// A loaded value, as the Python value or dataclass instance it projects to. +#[derive(Clone, Debug, PartialEq)] +pub enum Built { + /// A plain value, including `None`, booleans, integers and raw authored mappings. + Value(ConfigValue), + /// A dataclass from `sqlbuild.spec.contracts.models`; omitted fields keep their defaults. + Object(&'static str, Vec<(&'static str, Built)>), + /// A member of a named `StrEnum`, by value. + Enum(&'static str, String), + /// A `dict` with string keys, in insertion order. + Dict(Vec<(String, Built)>), + /// A `tuple` of strings. + Tuple(Vec), + /// A `frozenset` of strings. + FrozenSet(Vec), + /// `Decimal(str(value))` of an integer or float. + Decimal(ConfigValue), } diff --git a/crates/sqlbuild-config/src/project/tests.rs b/crates/sqlbuild-config/src/project/tests.rs index efd9060dc5..5371192a9c 100644 --- a/crates/sqlbuild-config/src/project/tests.rs +++ b/crates/sqlbuild-config/src/project/tests.rs @@ -1,6 +1,6 @@ #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_read_config.rs"] -mod read_config; +#[path = "tests/test_load_config.rs"] +mod load_config; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-config/src/project/tests/helpers.rs b/crates/sqlbuild-config/src/project/tests/helpers.rs index b7bb9560e6..92acfc56de 100644 --- a/crates/sqlbuild-config/src/project/tests/helpers.rs +++ b/crates/sqlbuild-config/src/project/tests/helpers.rs @@ -1,50 +1,46 @@ -use crate::models::ConfigValue; -use crate::project::models::{ProjectConfigFile, ProjectSettings}; +use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; +use crate::project::main::load_local_config::load_local_config; +use crate::project::main::load_project_config::load_project_config; +use crate::project::models::{ConfigFile, ConfigPaths}; use tempfile::TempDir; -/// A project directory containing `files`. -pub(super) fn project_directory(files: &[(&str, &str)]) -> TempDir { +/// A project directory with `project` and `local` files, and its config paths. +pub(super) fn project_directory(project: &str, local: &str) -> (TempDir, ConfigPaths) { let directory = TempDir::new().expect("temporary directory"); - for (name, contents) in files { - std::fs::write(directory.path().join(name), contents).expect("write project file"); - } - directory + std::fs::write(directory.path().join("sqlbuild_project.toml"), project).expect("project"); + std::fs::write(directory.path().join("sqlbuild_local.toml"), local).expect("local"); + let paths = ConfigPaths { + project: directory.path().join("sqlbuild_project.toml"), + legacy_project: directory.path().join("sqlbuild_project.yml"), + local: directory.path().join("sqlbuild_local.toml"), + legacy_local: directory.path().join("sqlbuild_local.yml"), + }; + (directory, paths) } -pub(super) fn minimal_project() -> ProjectConfigFile { - ProjectConfigFile { - name: "orders".to_owned(), - adapter: "duckdb".to_owned(), - default_target: None, - settings: ProjectSettings { - sql_analysis: true, - require_sql_analysis: false, - }, - enforce_placement: true, - enforce_explicit_references: true, - vars: vec![], - path_defaults: vec![], - target_names: vec![], - } +/// The first rejection of loading both files, if any. +pub(super) fn first_failure(paths: &ConfigPaths) -> Option { + [ + load_project_config(paths).err(), + load_local_config(paths).err(), + ] + .into_iter() + .flatten() + .find_map(LoadFailure::into_config_failure) } -pub(super) fn configured_project() -> ProjectConfigFile { - ProjectConfigFile { - default_target: Some("dev".to_owned()), - settings: ProjectSettings { - sql_analysis: false, - require_sql_analysis: true, - }, - enforce_placement: false, - vars: vec![("region".to_owned(), "eu".to_owned())], - path_defaults: vec![( - "staging".to_owned(), - ConfigValue::Map(vec![( - ConfigValue::String("materialized".to_owned()), - ConfigValue::String("view".to_owned()), - )]), - )], - target_names: vec!["dev".to_owned(), "prod".to_owned()], - ..minimal_project() - } +/// A rejection whose message starts with the path of `file`. +pub(super) fn failure_at( + file: ConfigFile, + text: &str, + help: Option<&str>, +) -> Option { + Some(ConfigFailure { + message: vec![ + MessagePiece::File(file), + MessagePiece::Text(format!(" {text}")), + ], + help: help.map(str::to_owned), + class: FailureClass::ProjectConfig, + }) } diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs new file mode 100644 index 0000000000..1f460c6232 --- /dev/null +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -0,0 +1,92 @@ +use crate::project::models::ConfigFile; +use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; +use crate::project::tests::test_types::ProjectFailureTestCase; + +const PROJECT: &str = "name = 'orders'\nadapter = 'duckdb'\n"; + +#[test] +fn given_project_files_when_loading_then_rejections_match_python_loader() { + let test_cases = [ + ProjectFailureTestCase { + description: "a valid project and local file load", + project: "name = 'orders'\nadapter = 'duckdb'\n[settings]\nconcurrency = 4\n\ + [targets.dev]\nschema = 'analytics'\n[sinks.lifecycle]\nshutdown_timeout = '10m'\n", + local: "target = 'dev'\n[settings]\nmax_concurrency = 2\n", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "an unknown key suggests the nearest supported key", + project: "name = 'orders'\nadapter = 'duckdb'\n[scopes]\nenforce_placment = true\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "scopes contains unknown key(s): enforce_placment. Allowed keys: enforce_placement", + Some("did you mean 'enforce_placement'?"), + ), + }, + ProjectFailureTestCase { + description: "the janitor requires query change tracking to delete tracked relations", + project: "name = 'orders'\nadapter = 'duckdb'\n[settings]\nquery_change_tracking = false\n\ + [janitor]\nenabled = true\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "[janitor] delete_tracked_only = true requires query change tracking; \ + sqlbuild_project.toml sets [settings] query_change_tracking = false", + Some( + "to track which relations SQLBuild created, set this in sqlbuild_project.toml:\n\ + \x20 [settings]\n query_change_tracking = true", + ), + ), + }, + ProjectFailureTestCase { + description: "a shutdown timeout over ten minutes is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[sinks.lifecycle]\nshutdown_timeout = '601s'\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "sinks.lifecycle.shutdown_timeout must be a fixed duration from '0s' to '10m' such as '30s'", + None, + ), + }, + ProjectFailureTestCase { + description: "a local file cannot set the shared SQL analysis policy", + project: PROJECT, + local: "[settings]\nrequire_sql_analysis = true\n", + expected_failure: failure_at( + ConfigFile::Local, + "cannot set [settings] require_sql_analysis; it is a shared project policy that \ + only sqlbuild_project.toml can set", + Some( + "remove require_sql_analysis from sqlbuild_local.toml\n = help: to change the \ + policy for everyone, set this in sqlbuild_project.toml:\n [settings]\n\ + \x20 require_sql_analysis = false", + ), + ), + }, + ProjectFailureTestCase { + description: "a path default normalizing to an existing key is rejected", + project: "name = 'orders'\nadapter = 'duckdb'\n[path_defaults.staging]\n[path_defaults.' staging ']\n", + local: "", + expected_failure: failure_at( + ConfigFile::Project, + "path_defaults defines both ' staging ' and another key that normalize to \ + 'staging'. Use only the canonical model-relative form without a leading \ + 'models/' prefix.", + None, + ), + }, + ]; + + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, test_case.local); + + let failure = first_failure(&paths); + + assert_eq!( + failure, test_case.expected_failure, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_read_config.rs b/crates/sqlbuild-config/src/project/tests/test_read_config.rs deleted file mode 100644 index 5d8534ec76..0000000000 --- a/crates/sqlbuild-config/src/project/tests/test_read_config.rs +++ /dev/null @@ -1,125 +0,0 @@ -use crate::errors::ConfigErrorKind; -use crate::project::main::read_local_config::read_local_config; -use crate::project::main::read_project_config::read_project_config; -use crate::project::models::{LocalConfigFile, ProjectSettings}; -use crate::project::tests::helpers::{configured_project, minimal_project, project_directory}; -use crate::project::tests::test_types::{LocalConfigTestCase, ProjectConfigTestCase}; - -const PROJECT: &str = "sqlbuild_project.toml"; -const LOCAL: &str = "sqlbuild_local.toml"; - -#[test] -fn given_project_files_when_reading_then_discovery_fields_match_python_loader() { - let test_cases = [ - ProjectConfigTestCase { - description: "defaults apply and strings are stripped", - files: &[(PROJECT, "name = ' orders '\nadapter = \"duckdb\"\n")], - expected_config: Ok(minimal_project()), - }, - ProjectConfigTestCase { - description: "discovery sections are read in document order", - files: &[( - PROJECT, - "name = 'orders'\nadapter = 'duckdb'\ndefault_target = 'dev'\n\ - [settings]\nsql_analysis = false\nrequire_sql_analysis = true\n\ - [scopes]\nenforce_placement = false\n[vars]\nregion = 'eu'\n\ - [path_defaults.staging]\nmaterialized = 'view'\n\ - [targets.dev]\n[targets.prod]\n", - )], - expected_config: Ok(configured_project()), - }, - ProjectConfigTestCase { - description: "legacy sql_validation sets SQL analysis", - files: &[( - PROJECT, - "name = 'orders'\nadapter = 'duckdb'\n[settings]\nsql_validation = false\n", - )], - expected_config: Ok(crate::project::models::ProjectConfigFile { - settings: ProjectSettings { - sql_analysis: false, - require_sql_analysis: false, - }, - ..minimal_project() - }), - }, - ProjectConfigTestCase { - description: "conflicting analysis keys are rejected", - files: &[( - PROJECT, - "name = 'orders'\nadapter = 'duckdb'\n[settings]\nsql_analysis = true\nsql_validation = false\n", - )], - expected_config: Err(ConfigErrorKind::InvalidField), - }, - ProjectConfigTestCase { - description: "blank name is rejected", - files: &[(PROJECT, "name = ' '\nadapter = 'duckdb'\n")], - expected_config: Err(ConfigErrorKind::InvalidField), - }, - ProjectConfigTestCase { - description: "non-string var is rejected", - files: &[( - PROJECT, - "name = 'orders'\nadapter = 'duckdb'\n[vars]\nlimit = 3\n", - )], - expected_config: Err(ConfigErrorKind::InvalidField), - }, - ProjectConfigTestCase { - description: "legacy YAML project file is rejected", - files: &[("sqlbuild_project.yml", "name: orders\n")], - expected_config: Err(ConfigErrorKind::InvalidField), - }, - ProjectConfigTestCase { - description: "missing project file", - files: &[], - expected_config: Err(ConfigErrorKind::Missing), - }, - ]; - for test_case in test_cases { - let directory = project_directory(test_case.files); - let actual = read_project_config(directory.path()).map_err(|error| error.kind); - assert_eq!( - actual, test_case.expected_config, - "{}", - test_case.description - ); - } -} - -#[test] -fn given_local_files_when_reading_then_overrides_match_python_loader() { - let test_cases = [ - LocalConfigTestCase { - description: "absent local file reads as defaults", - files: &[], - expected_config: Ok(LocalConfigFile::default()), - }, - LocalConfigTestCase { - description: "local overrides are read", - files: &[( - LOCAL, - "target = 'dev'\n[settings]\nsql_validation = false\n[vars]\nregion = 'us'\n[targets.dev]\n", - )], - expected_config: Ok(LocalConfigFile { - target: Some("dev".to_owned()), - adapter: None, - sql_analysis: Some(false), - vars: vec![("region".to_owned(), "us".to_owned())], - target_names: vec!["dev".to_owned()], - }), - }, - LocalConfigTestCase { - description: "legacy YAML local file is rejected", - files: &[("sqlbuild_local.yml", "target: dev\n")], - expected_config: Err(ConfigErrorKind::InvalidField), - }, - ]; - for test_case in test_cases { - let directory = project_directory(test_case.files); - let actual = read_local_config(directory.path()).map_err(|error| error.kind); - assert_eq!( - actual, test_case.expected_config, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index f234c337c6..bd640f62a8 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -1,14 +1,8 @@ -use crate::errors::ConfigErrorKind; -use crate::project::models::{LocalConfigFile, ProjectConfigFile}; +use crate::project::errors::ConfigFailure; -pub(super) struct ProjectConfigTestCase { +pub(super) struct ProjectFailureTestCase { pub(super) description: &'static str, - pub(super) files: &'static [(&'static str, &'static str)], - pub(super) expected_config: Result, -} - -pub(super) struct LocalConfigTestCase { - pub(super) description: &'static str, - pub(super) files: &'static [(&'static str, &'static str)], - pub(super) expected_config: Result, + pub(super) project: &'static str, + pub(super) local: &'static str, + pub(super) expected_failure: Option, } diff --git a/crates/sqlbuild-config/src/project/types.rs b/crates/sqlbuild-config/src/project/types.rs new file mode 100644 index 0000000000..821240406f --- /dev/null +++ b/crates/sqlbuild-config/src/project/types.rs @@ -0,0 +1,11 @@ +//! Shared shapes of project configuration loading. + +use crate::models::ConfigValue; +use crate::project::errors::ConfigFailure; + +/// The entries of a loaded mapping. +pub type Entries = [(ConfigValue, ConfigValue)]; +/// A loading step's result. +pub type Loaded = Result; +/// Dataclass fields by name. +pub type Fields = Vec<(&'static str, crate::project::models::Built)>; diff --git a/crates/sqlbuild-config/src/toml/_helpers/integers.rs b/crates/sqlbuild-config/src/toml/_helpers/integers.rs new file mode 100644 index 0000000000..06ca798e0a --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/integers.rs @@ -0,0 +1,121 @@ +//! Values the TOML parser rejects but `tomllib` reads: integers beyond 64 bits and deep nesting. + +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::{ConfigValue, Natural}; + +const DECIMAL_RADIX: u32 = 10; +/// Noncharacters that can mark a placeholder; the first absent from a document is used. +const PLACEHOLDER_MARKS: std::ops::RangeInclusive = 0xFDD0..=0xFDEF; + +/// A value read apart from the document: its byte range there and the value it stands for. +pub(crate) type Placeholder = (usize, usize, ConfigValue); + +fn digits_with_separators(body: &str, radix: u32) -> Option> { + let mut values: Vec = Vec::new(); + let mut after_digit = false; + for character in body.chars() { + if character == '_' && after_digit { + after_digit = false; + continue; + } + values.push(character.to_digit(radix)?); + after_digit = true; + } + after_digit.then_some(values) +} + +fn natural(values: &[u32], radix: u32) -> String { + values + .iter() + .fold(Natural::default(), |total, value| { + total.times_plus(radix, *value) + }) + .decimal() +} + +/// The signed decimal digits of a TOML integer literal outside the 64-bit range, if `raw` is one. +pub(crate) fn large_integer(raw: &str) -> Option { + let (radix, body, negative) = match raw.get(..2) { + Some("0x") => (16, &raw[2..], false), + Some("0o") => (8, &raw[2..], false), + Some("0b") => (2, &raw[2..], false), + _ => match raw.strip_prefix('-') { + Some(rest) => (10, rest, true), + None => (10, raw.strip_prefix('+').unwrap_or(raw), false), + }, + }; + let values = digits_with_separators(body, radix)?; + if radix == DECIMAL_RADIX && values.len() > 1 && values[0] == 0 { + return None; + } + let digits = natural(&values, radix); + let signed = if negative { + format!("-{digits}") + } else { + digits + }; + signed.parse::().is_err().then_some(signed) +} + +/// A placeholder mark that neither appears in `text` nor can come from one of its escapes. +pub(crate) fn placeholder_mark(text: &str) -> Result { + let lowered = text.to_ascii_lowercase(); + PLACEHOLDER_MARKS + .filter_map(char::from_u32) + .find(|mark| !text.contains(*mark) && !lowered.contains(&format!("{:x}", u32::from(*mark)))) + .ok_or_else(|| { + ConfigError::new( + ConfigErrorKind::Unsupported, + "large integers in text using every placeholder mark are not read natively", + ) + }) +} + +/// `text` with each placeholder range replaced by a string marked by `mark`. +pub(crate) fn substitute_placeholders(text: &str, integers: &[Placeholder], mark: char) -> String { + let mut substituted = String::with_capacity(text.len()); + let mut copied = 0; + for (index, (start, end, _)) in integers.iter().enumerate() { + substituted.push_str(&text[copied..*start]); + substituted.push_str(&format!("\"{mark}{index}\"")); + copied = *end; + } + substituted.push_str(&text[copied..]); + substituted +} + +/// `value` with each placeholder string turned back into the value it stands for. +pub(crate) fn restore_placeholders( + value: ConfigValue, + integers: &[Placeholder], + mark: char, +) -> ConfigValue { + match value { + ConfigValue::String(text) => match text + .strip_prefix(mark) + .and_then(|index| { + integers + .iter() + .enumerate() + .find(|(position, _)| position.to_string() == index) + }) + .map(|(_, integer)| integer) + { + Some((_, _, restored)) => restored.clone(), + None => ConfigValue::String(text), + }, + ConfigValue::List(items) => ConfigValue::List( + items + .into_iter() + .map(|item| restore_placeholders(item, integers, mark)) + .collect(), + ), + ConfigValue::Map(entries) => ConfigValue::Map( + entries + .into_iter() + .map(|(key, item)| (key, restore_placeholders(item, integers, mark))) + .collect(), + ), + other => other, + } +} diff --git a/crates/sqlbuild-config/src/toml/_helpers/mod.rs b/crates/sqlbuild-config/src/toml/_helpers/mod.rs index cd76bd6968..eacec567c1 100644 --- a/crates/sqlbuild-config/src/toml/_helpers/mod.rs +++ b/crates/sqlbuild-config/src/toml/_helpers/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod conversion; +pub(crate) mod integers; pub(crate) mod namespaces; +pub(crate) mod reading; pub(crate) mod version; diff --git a/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs b/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs index 9961ccba4f..afecd22a49 100644 --- a/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs +++ b/crates/sqlbuild-config/src/toml/_helpers/namespaces.rs @@ -1,6 +1,7 @@ //! `tomllib`'s namespace rules: which tables, dotted keys and arrays of tables may be (re)opened. use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::toml::constants::MAX_CONTAINER_DEPTH; use std::collections::HashMap; use toml_parser::decoder::Encoding; use toml_parser::parser::{EventReceiver, RecursionGuard, parse_document}; @@ -10,9 +11,6 @@ use toml_parser::{ErrorSink, ParseError, Raw, Source, Span}; const FROZEN: u8 = 1; /// `Flags.EXPLICIT_NEST`: a table declared by a header or opened by a dotted key. const EXPLICIT_NEST: u8 = 2; -/// The container depth the version check already enforces. -const MAX_CONTAINER_DEPTH: u32 = 80; - /// One key of `tomllib`'s `Flags` tree. #[derive(Default)] struct FlagNode { diff --git a/crates/sqlbuild-config/src/toml/_helpers/reading.rs b/crates/sqlbuild-config/src/toml/_helpers/reading.rs new file mode 100644 index 0000000000..abb2cea2ee --- /dev/null +++ b/crates/sqlbuild-config/src/toml/_helpers/reading.rs @@ -0,0 +1,78 @@ +//! Read a TOML document natively, splitting off what the parser cannot nest. + +use crate::constants::BYTE_ORDER_MARK; +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::toml::_helpers::integers::{ + Placeholder, placeholder_mark, restore_placeholders, substitute_placeholders, +}; +use crate::toml::constants::SPLIT_VALUE_KEY; + +/// The value of one container nested past the parser's limit, read as its own document. +fn deep_value(text: &str, (start, end): (usize, usize)) -> Result { + let document = format!("{SPLIT_VALUE_KEY} = {}", &text[start..end]); + let lines_before = text[..start].matches('\n').count(); + let value = load_document(&document).map_err(|error| ConfigError { + line: error.line.map(|line| line + lines_before), + ..error + })?; + let restored = value + .get(SPLIT_VALUE_KEY) + .cloned() + .unwrap_or(ConfigValue::Null); + Ok((start, end, restored)) +} + +fn is_inside(ranges: &[(usize, usize)], offset: usize) -> bool { + ranges + .iter() + .any(|(start, end)| *start <= offset && offset < *end) +} + +/// An upper bound of the bracket nesting of `text`, counting brackets inside strings too. +pub(crate) fn bracket_depth(text: &str) -> usize { + let mut depth: usize = 0; + let mut deepest: usize = 0; + for byte in text.bytes() { + match byte { + b'[' | b'{' => { + depth += 1; + deepest = deepest.max(depth); + } + b']' | b'}' => depth = depth.saturating_sub(1), + _ => {} + } + } + deepest +} + +/// `tomllib.loads` on the current thread. +pub(crate) fn load_document(text: &str) -> Result { + if text.starts_with(BYTE_ORDER_MARK) { + return Err(ConfigError::new( + ConfigErrorKind::Syntax, + "invalid statement: tomllib does not skip a byte order mark", + )); + } + let text = text.replace("\r\n", "\n"); + let (large_integers, deep_ranges) = + crate::toml::_helpers::version::reject_toml_1_1_syntax(&text)?; + crate::toml::_helpers::namespaces::reject_namespace_conflicts(&text)?; + let mut placeholders: Vec = deep_ranges + .iter() + .map(|range| deep_value(&text, *range)) + .collect::>()?; + placeholders.extend( + large_integers + .into_iter() + .filter(|(start, _, _)| !is_inside(&deep_ranges, *start)), + ); + if placeholders.is_empty() { + return crate::toml::_helpers::conversion::parse_document(&text); + } + placeholders.sort_by_key(|(start, _, _)| *start); + let mark = placeholder_mark(&text)?; + let substituted = substitute_placeholders(&text, &placeholders, mark); + crate::toml::_helpers::conversion::parse_document(&substituted) + .map(|value| restore_placeholders(value, &placeholders, mark)) +} diff --git a/crates/sqlbuild-config/src/toml/_helpers/version.rs b/crates/sqlbuild-config/src/toml/_helpers/version.rs index 008be45309..2d13536ef7 100644 --- a/crates/sqlbuild-config/src/toml/_helpers/version.rs +++ b/crates/sqlbuild-config/src/toml/_helpers/version.rs @@ -1,12 +1,16 @@ //! Find TOML 1.1 syntax, which the parser accepts and Python's TOML 1.0 `tomllib` rejects. use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::ConfigValue; +use crate::toml::_helpers::integers::{Placeholder, large_integer}; +use crate::toml::constants::{MAX_CONTAINER_DEPTH, SPLIT_CONTAINER_DEPTH}; use toml_parser::decoder::Encoding; use toml_parser::parser::{EventReceiver, RecursionGuard, parse_document}; use toml_parser::{ErrorSink, ParseError, Source, Span}; -/// The container depth `toml_edit` accepts; deeper documents are left to Python. -const MAX_CONTAINER_DEPTH: u32 = 80; +/// Large integers, and the byte ranges of containers nested at the split depth. +pub(crate) type NativeSplits = (Vec, Vec<(usize, usize)>); + /// The most dotted key segments read natively, which bounds the table nesting of a document. const MAX_KEY_SEGMENTS: usize = 128; @@ -24,6 +28,9 @@ struct VersionReceiver<'text> { after_comma: bool, key_segments: usize, found: Option<(usize, &'static str)>, + large_integers: Vec, + deep_values: Vec<(usize, usize)>, + deep_start: usize, } impl VersionReceiver<'_> { @@ -50,6 +57,32 @@ impl VersionReceiver<'_> { } } + fn record_large_integer(&mut self, span: Span) { + let raw = self + .source + .get(span) + .map(|raw| raw.as_str()) + .unwrap_or_default(); + if let Some(digits) = large_integer(raw) { + self.large_integers + .push((span.start(), span.end(), ConfigValue::BigInteger(digits))); + } + } + + fn open_container(&mut self, span: Span, container: Container) { + self.containers.push(container); + if self.containers.len() == SPLIT_CONTAINER_DEPTH { + self.deep_start = span.start(); + } + } + + fn close_container(&mut self, span: Span) { + if self.containers.len() == SPLIT_CONTAINER_DEPTH { + self.deep_values.push((self.deep_start, span.end())); + } + self.containers.pop(); + } + /// Escapes added by TOML 1.1: `\e` and `\xHH`. fn check_escapes(&mut self, span: Span, encoding: Option) { if !matches!( @@ -82,9 +115,9 @@ impl EventReceiver for VersionReceiver<'_> { self.end_key(); } - fn inline_table_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + fn inline_table_open(&mut self, span: Span, _error: &mut dyn ErrorSink) -> bool { self.end_key(); - self.containers.push(Container::InlineTable); + self.open_container(span, Container::InlineTable); true } @@ -93,18 +126,18 @@ impl EventReceiver for VersionReceiver<'_> { self.record(span, "a trailing comma in an inline table"); } self.token(); - self.containers.pop(); + self.close_container(span); } - fn array_open(&mut self, _span: Span, _error: &mut dyn ErrorSink) -> bool { + fn array_open(&mut self, span: Span, _error: &mut dyn ErrorSink) -> bool { self.token(); - self.containers.push(Container::Array); + self.open_container(span, Container::Array); true } - fn array_close(&mut self, _span: Span, _error: &mut dyn ErrorSink) { + fn array_close(&mut self, span: Span, _error: &mut dyn ErrorSink) { self.token(); - self.containers.pop(); + self.close_container(span); } fn simple_key(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { @@ -127,6 +160,9 @@ impl EventReceiver for VersionReceiver<'_> { fn scalar(&mut self, span: Span, encoding: Option, _error: &mut dyn ErrorSink) { self.token(); self.check_escapes(span, encoding); + if encoding.is_none() { + self.record_large_integer(span); + } } fn value_sep(&mut self, _span: Span, _error: &mut dyn ErrorSink) { @@ -144,8 +180,8 @@ impl EventReceiver for VersionReceiver<'_> { } } -/// Reject TOML 1.1-only syntax, syntax errors, and nesting beyond the native limits. -pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result<(), ConfigError> { +/// Reject TOML 1.1 and syntax errors; return large integers and values nested past the parser. +pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result { let source = Source::new(text); let tokens = source.lex().into_vec(); let mut receiver = VersionReceiver { @@ -154,6 +190,9 @@ pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result<(), ConfigError> { after_comma: false, key_segments: 0, found: None, + large_integers: Vec::new(), + deep_values: Vec::new(), + deep_start: 0, }; let mut errors: Vec = Vec::new(); parse_document( @@ -171,7 +210,7 @@ pub(crate) fn reject_toml_1_1_syntax(text: &str) -> Result<(), ConfigError> { }); } match receiver.found { - None => Ok(()), + None => Ok((receiver.large_integers, receiver.deep_values)), Some((offset, construct)) => Err(ConfigError { kind: ConfigErrorKind::Syntax, message: format!("{construct} is TOML 1.1 syntax, which tomllib rejects"), diff --git a/crates/sqlbuild-config/src/toml/constants.rs b/crates/sqlbuild-config/src/toml/constants.rs new file mode 100644 index 0000000000..24f1b965a6 --- /dev/null +++ b/crates/sqlbuild-config/src/toml/constants.rs @@ -0,0 +1,12 @@ +//! Native TOML nesting limits. + +/// The deepest inline nesting read natively; `tomllib` exhausts Python's recursion limit by 497. +pub(crate) const MAX_CONTAINER_DEPTH: u32 = 512; +/// Inline containers at this depth are read as separate values, under the parser's own limit. +pub(crate) const SPLIT_CONTAINER_DEPTH: usize = 64; +/// The key a separately read value is assigned to. +pub(crate) const SPLIT_VALUE_KEY: &str = "value"; +/// Bracket depth past which a document is read on a thread with a larger stack. +pub(crate) const DEEP_DOCUMENT_BRACKETS: usize = 128; +/// The stack of that thread: the recursive parser needs more than a default thread has. +pub(crate) const DEEP_DOCUMENT_STACK_BYTES: usize = 64 * 1024 * 1024; diff --git a/crates/sqlbuild-config/src/toml/main/load_toml.rs b/crates/sqlbuild-config/src/toml/main/load_toml.rs index 9e7ab66184..7f0120c8c2 100644 --- a/crates/sqlbuild-config/src/toml/main/load_toml.rs +++ b/crates/sqlbuild-config/src/toml/main/load_toml.rs @@ -1,19 +1,26 @@ //! Load a TOML document the way Python's `tomllib.loads` does. -use crate::constants::BYTE_ORDER_MARK; use crate::errors::{ConfigError, ConfigErrorKind}; use crate::models::ConfigValue; +use crate::toml::_helpers::reading::{bracket_depth, load_document}; +use crate::toml::constants::{DEEP_DOCUMENT_BRACKETS, DEEP_DOCUMENT_STACK_BYTES}; /// The table `tomllib.loads(text)` returns, rejecting TOML 1.1 syntax as `tomllib` does. pub fn load_toml(text: &str) -> Result { - if text.starts_with(BYTE_ORDER_MARK) { - return Err(ConfigError::new( - ConfigErrorKind::Syntax, - "invalid statement: tomllib does not skip a byte order mark", - )); + if bracket_depth(text) <= DEEP_DOCUMENT_BRACKETS { + return load_document(text); } - let text = text.replace("\r\n", "\n"); - crate::toml::_helpers::version::reject_toml_1_1_syntax(&text)?; - crate::toml::_helpers::namespaces::reject_namespace_conflicts(&text)?; - crate::toml::_helpers::conversion::parse_document(&text) + std::thread::scope(|scope| { + std::thread::Builder::new() + .stack_size(DEEP_DOCUMENT_STACK_BYTES) + .spawn_scoped(scope, || load_document(text)) + .map_err(|error| ConfigError::new(ConfigErrorKind::Unsupported, error.to_string()))? + .join() + .unwrap_or_else(|_| { + Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "the deep TOML reader stopped unexpectedly", + )) + }) + }) } diff --git a/crates/sqlbuild-config/src/toml/mod.rs b/crates/sqlbuild-config/src/toml/mod.rs index 668a17d0aa..37f9081f4f 100644 --- a/crates/sqlbuild-config/src/toml/mod.rs +++ b/crates/sqlbuild-config/src/toml/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; #[cfg(test)] diff --git a/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs b/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs index 078edffbd4..b32405b423 100644 --- a/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs +++ b/crates/sqlbuild-config/src/toml/tests/test_load_toml.rs @@ -2,7 +2,7 @@ use crate::errors::ConfigErrorKind; use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; use crate::toml::main::load_toml::load_toml; use crate::toml::tests::helpers::table; -use crate::toml::tests::test_types::{HostileTomlTestCase, LoadTomlTestCase}; +use crate::toml::tests::test_types::{DeepTomlTestCase, HostileTomlTestCase, LoadTomlTestCase}; #[test] fn given_toml_documents_when_loading_then_values_match_tomllib() { @@ -28,6 +28,43 @@ fn given_toml_documents_when_loading_then_values_match_tomllib() { ), ])), }, + LoadTomlTestCase { + description: "integers beyond 64 bits are read exactly in every base and position", + text: "a = 18_446_744_073_709_551_616\nb = -9223372036854775809\n\ + c = [0xFFFF_FFFF_FFFF_FFFF, 0o7777777777777777777777]\nd = {e = 0b1_0000000000000000000000000000000000000000000000000000000000000000}\n\ + f = '\u{fdd0}0'\ng = 9223372036854775807", + expected_value: Ok(table(vec![ + ( + "a", + ConfigValue::BigInteger("18446744073709551616".to_owned()), + ), + ( + "b", + ConfigValue::BigInteger("-9223372036854775809".to_owned()), + ), + ( + "c", + ConfigValue::List(vec![ + ConfigValue::BigInteger("18446744073709551615".to_owned()), + ConfigValue::BigInteger("73786976294838206463".to_owned()), + ]), + ), + ( + "d", + table(vec![( + "e", + ConfigValue::BigInteger("18446744073709551616".to_owned()), + )]), + ), + ("f", ConfigValue::String("\u{fdd0}0".to_owned())), + ("g", ConfigValue::Integer(i64::MAX)), + ])), + }, + LoadTomlTestCase { + description: "an invalid integer literal stays a syntax error", + text: "a = 1__000000000000000000000", + expected_value: Err(ConfigErrorKind::Syntax), + }, LoadTomlTestCase { description: "implicit parent tables are created where first named", text: "[a.b]\nx = 1\n[c]\n[a.d]\ny = 2", @@ -328,3 +365,41 @@ fn given_hostile_toml_when_loading_then_native_reader_rejects_without_exhausting ); } } + +#[test] +fn given_values_nested_past_the_parser_limit_when_loading_then_tomllib_values_are_kept() { + let test_cases = [ + DeepTomlTestCase { + description: "two hundred nested arrays", + text: format!("a = {}1{}\nb = 2", "[".repeat(200), "]".repeat(200)), + expected_value: (0..200).fold(ConfigValue::Integer(1), |inner, _| { + ConfigValue::List(vec![inner]) + }), + }, + DeepTomlTestCase { + description: "two hundred nested inline tables holding a large integer", + text: format!( + "a = {}99999999999999999999{}\nb = 2", + "{b = ".repeat(200), + "}".repeat(200) + ), + expected_value: (0..199).fold( + table(vec![( + "b", + ConfigValue::BigInteger("99999999999999999999".to_owned()), + )]), + |inner, _| table(vec![("b", inner)]), + ), + }, + ]; + for test_case in test_cases { + let loaded = load_toml(&test_case.text).map(|value| value.get("a").cloned()); + + assert_eq!( + loaded, + Ok(Some(test_case.expected_value)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/toml/tests/test_types.rs b/crates/sqlbuild-config/src/toml/tests/test_types.rs index cb0c0f42cf..c13058e260 100644 --- a/crates/sqlbuild-config/src/toml/tests/test_types.rs +++ b/crates/sqlbuild-config/src/toml/tests/test_types.rs @@ -12,3 +12,9 @@ pub(super) struct HostileTomlTestCase { pub(super) text: String, pub(super) expected_error: ConfigErrorKind, } + +pub(super) struct DeepTomlTestCase { + pub(super) description: &'static str, + pub(super) text: String, + pub(super) expected_value: ConfigValue, +} diff --git a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs index 130d68753f..5b7d7785b9 100644 --- a/crates/sqlbuild-config/src/yaml/_helpers/composer.rs +++ b/crates/sqlbuild-config/src/yaml/_helpers/composer.rs @@ -10,7 +10,7 @@ use crate::yaml::constants::{ FLOW_KEY_INDICATOR, MAP_TAG, MAX_NATIVE_IMPLICIT_KEY_CHARACTERS, MAX_NESTING_DEPTH, NON_SPECIFIC_TAG, PLAIN_FORBIDDEN_FIRST_CHARACTERS, SEQ_TAG, SEQUENCE_ENTRY_TOKEN, }; -use crate::yaml::models::{ComposedDocument, Node, NodeContent}; +use crate::yaml::models::{ComposedDocument, MarkStart, Node, NodeContent}; use saphyr_parser::{Event, Parser, ScalarStyle, ScanError, Span, StrInput, Tag}; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; @@ -55,7 +55,9 @@ struct Composer<'input> { anchors: HashMap, anchor_names: HashSet, open_anchors: HashSet, + previous_start: usize, previous_end: usize, + current_start: usize, current_end: usize, } @@ -85,7 +87,9 @@ impl<'input> Composer<'input> { fn next(&mut self) -> Result<(Event<'input>, Span), ConfigError> { match self.parser.next_event() { Some(Ok(event)) => { + self.previous_start = self.current_start; self.previous_end = self.current_end; + self.current_start = event.1.start.index(); self.current_end = event.1.end.index(); Ok(event) } @@ -97,11 +101,19 @@ impl<'input> Composer<'input> { } } - fn push(&mut self, tag: String, content: NodeContent, span: Span) -> usize { + fn push( + &mut self, + tag: String, + content: NodeContent, + span: Span, + mark_start: MarkStart, + ) -> usize { self.document.nodes.push(Node { tag, content, position: start_position(span), + span: (span.start.index(), span.end.index()), + mark_start, }); self.document.nodes.len() - 1 } @@ -262,6 +274,68 @@ impl<'input> Composer<'input> { None } + /// PyYAML's start mark: the first property after the last event or in an implicit start. + fn mark_start(&self, has_properties: bool, span: Span) -> MarkStart { + if has_properties { + let start: usize = span.start.index(); + self.first_property(self.previous_end, start) + .or_else(|| self.first_property(self.previous_start, start)) + .map_or(MarkStart::Unknown, MarkStart::Property) + } else { + MarkStart::Token + } + } + + /// The first `&` or `!` token from `from` to `start`, skipping comments. + fn first_property(&self, from: usize, start: usize) -> Option { + let mut index = from.min(start); + let mut token_start = true; + while index < start { + let character = self.char_at(index)?; + let after_space = + index == 0 || self.char_at(index - 1).is_some_and(char::is_whitespace); + if character == '#' && after_space { + while !matches!(self.char_at(index), None | Some('\n' | '\r')) { + index += 1; + } + continue; + } + if matches!(character, '&' | '!') && token_start { + return Some(index); + } + token_start = character.is_whitespace() || matches!(character, '[' | '{' | ','); + index += 1; + } + None + } + + /// Just past the closing quote of the quoted scalar whose opening quote is at `start`. + fn quoted_end(&self, start: usize) -> Option { + let quote = self.char_at(start)?; + let mut index = start + 1; + loop { + let character = self.char_at(index)?; + match (quote, character) { + ('\'', '\'') if self.char_at(index + 1) == Some('\'') => index += 2, + ('"', '\\') => index += 2, + (_, character) if character == quote => return Some(index + 1), + _ => index += 1, + } + } + } + + /// PyYAML's block scalar marks: indicator to just past the last consumed line break. + fn block_marks(&self, span: Span) -> (usize, usize) { + let start = self.block_indicator(span).unwrap_or(span.start.index()); + let end = span.end.index().min(self.chars.len()); + let mut trimmed = end; + while trimmed > start && self.chars[trimmed - 1] == ' ' { + trimmed -= 1; + } + let after_break = trimmed > start && matches!(self.chars[trimmed - 1], '\n' | '\r'); + (start, if after_break { trimmed } else { end }) + } + /// The column of the `-` that starts the sequence entry holding the indicator at `indicator`. fn entry_column(&self, indicator: usize) -> Option { let mut end = indicator; @@ -372,19 +446,34 @@ impl<'input> Composer<'input> { if style == ScalarStyle::Plain && tag.is_none() && anchor_id == 0 { self.check_plain(&value, span, placement)?; } - let value = match style { - ScalarStyle::Literal | ScalarStyle::Folded => { - Cow::Owned(self.block_scalar(&value, span, placement)?) + let (value, marks) = match style { + ScalarStyle::Literal | ScalarStyle::Folded => ( + Cow::Owned(self.block_scalar(&value, span, placement)?), + self.block_marks(span), + ), + ScalarStyle::SingleQuoted | ScalarStyle::DoubleQuoted => { + let start = span.start.index(); + ( + value, + (start, self.quoted_end(start).unwrap_or(span.end.index())), + ) } - _ => value, + _ => (value, (span.start.index(), span.end.index())), }; let non_specific = full_tag(tag.as_ref()).as_deref() == Some(NON_SPECIFIC_TAG); if non_specific && value.is_empty() && style == ScalarStyle::Plain { return Err(unsupported("an empty scalar tagged '!'")); } self.open_anchor(anchor_id, span)?; + let mark_start = self.mark_start(anchor_id != 0 || tag.is_some(), span); let tag = scalar_tag(&value, style, full_tag(tag.as_ref())); - let node = self.push(tag, NodeContent::Scalar(value.into_owned()), span); + let node = self.push( + tag, + NodeContent::Scalar(value.into_owned()), + span, + mark_start, + ); + self.document.nodes[node].span = marks; Ok(self.close_anchor(anchor_id, node)) } Event::SequenceStart(anchor_id, tag) => { @@ -404,6 +493,7 @@ impl<'input> Composer<'input> { placement: Placement, ) -> Result { let (anchor_id, tag) = properties; + let mark_start = self.mark_start(anchor_id != 0 || tag.is_some(), span); self.open_anchor(anchor_id, span)?; let item_placement = placement.child(self.is_flow(span), false, Parent::SequenceEntry); let mut items: Vec = Vec::new(); @@ -415,7 +505,7 @@ impl<'input> Composer<'input> { items.push(self.compose_node(event, item_span, item_placement)?); } let tag = collection_tag(full_tag(tag.as_ref()), SEQ_TAG); - let node = self.push(tag, NodeContent::Sequence(items), span); + let node = self.push(tag, NodeContent::Sequence(items), span, mark_start); Ok(self.close_anchor(anchor_id, node)) } @@ -426,6 +516,7 @@ impl<'input> Composer<'input> { placement: Placement, ) -> Result { let (anchor_id, tag) = properties; + let mark_start = self.mark_start(anchor_id != 0 || tag.is_some(), span); self.open_anchor(anchor_id, span)?; let flow = self.is_flow(span); let mut entries: Vec<(usize, usize)> = Vec::new(); @@ -443,7 +534,7 @@ impl<'input> Composer<'input> { entries.push((key, value)); } let tag = collection_tag(full_tag(tag.as_ref()), MAP_TAG); - let node = self.push(tag, NodeContent::Mapping(entries), span); + let node = self.push(tag, NodeContent::Mapping(entries), span, mark_start); Ok(self.close_anchor(anchor_id, node)) } @@ -502,7 +593,9 @@ pub(crate) fn compose(text: &str) -> Result { anchors: HashMap::new(), anchor_names: HashSet::new(), open_anchors: HashSet::new(), + previous_start: 0, previous_end: 0, + current_start: 0, current_end: 0, } .compose_stream() diff --git a/crates/sqlbuild-config/src/yaml/main/compose_marks.rs b/crates/sqlbuild-config/src/yaml/main/compose_marks.rs new file mode 100644 index 0000000000..dc03a2e462 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/main/compose_marks.rs @@ -0,0 +1,45 @@ +//! Compose one YAML document the way PyYAML's `yaml.compose` does, keeping node marks. + +use crate::constants::BYTE_ORDER_MARK; +use crate::errors::{ConfigError, ConfigErrorKind}; +use crate::models::{ComposedYaml, ComposedYamlContent, ComposedYamlNode}; +use crate::yaml::models::{MarkStart, Node, NodeContent}; + +/// PyYAML's safe `yaml.compose` nodes; scalar marks are PyYAML's char-index marks. +pub fn compose_marks(text: &str) -> Result { + let shift = usize::from(text.starts_with(BYTE_ORDER_MARK)); + let text = crate::yaml::_helpers::reader::without_byte_order_mark(text); + crate::yaml::_helpers::reader::check_characters(text)?; + let document = crate::yaml::_helpers::composer::compose(text)?; + let nodes: Vec = document + .nodes + .into_iter() + .map(|node| composed_node(node, shift)) + .collect::>()?; + Ok(ComposedYaml { + nodes, + root: document.root, + }) +} + +fn composed_node(node: Node, shift: usize) -> Result { + let start: usize = match node.mark_start { + MarkStart::Token => node.span.0, + MarkStart::Property(start) => start, + MarkStart::Unknown => { + return Err(ConfigError::new( + ConfigErrorKind::Unsupported, + "node properties PyYAML marks differently", + )); + } + }; + Ok(ComposedYamlNode { + start: start + shift, + end: node.span.1 + shift, + content: match node.content { + NodeContent::Scalar(value) => ComposedYamlContent::Scalar(value), + NodeContent::Sequence(items) => ComposedYamlContent::Sequence(items), + NodeContent::Mapping(entries) => ComposedYamlContent::Mapping(entries), + }, + }) +} diff --git a/crates/sqlbuild-config/src/yaml/main/mod.rs b/crates/sqlbuild-config/src/yaml/main/mod.rs index ed61b475fd..86295c6255 100644 --- a/crates/sqlbuild-config/src/yaml/main/mod.rs +++ b/crates/sqlbuild-config/src/yaml/main/mod.rs @@ -1 +1,2 @@ +pub mod compose_marks; pub mod safe_load; diff --git a/crates/sqlbuild-config/src/yaml/models.rs b/crates/sqlbuild-config/src/yaml/models.rs index 36655fb837..7de8b9a373 100644 --- a/crates/sqlbuild-config/src/yaml/models.rs +++ b/crates/sqlbuild-config/src/yaml/models.rs @@ -15,6 +15,10 @@ pub(crate) struct Node { pub(crate) content: NodeContent, /// The one-based line and column where the node starts. pub(crate) position: (usize, usize), + /// The char indices where the node's token starts and ends, as the parser marks them. + pub(crate) span: (usize, usize), + /// Where PyYAML's start mark is when the node has properties. + pub(crate) mark_start: MarkStart, } /// A composed document; `root` is `None` for an empty stream. @@ -23,3 +27,11 @@ pub(crate) struct ComposedDocument { pub(crate) nodes: Vec, pub(crate) root: Option, } + +/// PyYAML's start mark of a node: its token, its first property, or unknown. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum MarkStart { + Token, + Property(usize), + Unknown, +} diff --git a/crates/sqlbuild-config/src/yaml/tests.rs b/crates/sqlbuild-config/src/yaml/tests.rs index b1224a8c91..4a11cae9d7 100644 --- a/crates/sqlbuild-config/src/yaml/tests.rs +++ b/crates/sqlbuild-config/src/yaml/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_compose_marks.rs"] +mod compose_marks; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_safe_load.rs"] diff --git a/crates/sqlbuild-config/src/yaml/tests/helpers.rs b/crates/sqlbuild-config/src/yaml/tests/helpers.rs index 13097546b3..4a2d9796d9 100644 --- a/crates/sqlbuild-config/src/yaml/tests/helpers.rs +++ b/crates/sqlbuild-config/src/yaml/tests/helpers.rs @@ -1,4 +1,6 @@ -use crate::models::{ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use crate::errors::ConfigErrorKind; +use crate::models::{ComposedYamlNode, ConfigDate, ConfigDateTime, ConfigTime, ConfigValue}; +use crate::yaml::main::compose_marks::compose_marks; pub(super) fn text(value: &str) -> ConfigValue { ConfigValue::String(value.to_owned()) @@ -71,3 +73,32 @@ pub(super) fn deep_anchor_chain(links: usize, depth: usize) -> String { format!("{text}a{link}: &a{link} {open}*a{}{close}\n", link - 1) }) } + +/// `(start, end, value)` of every scalar node, sorted, or the composer's error kind. +pub(super) fn scalar_marks(text: &str) -> Result, ConfigErrorKind> { + compose_marks(text) + .map(|document| { + let mut found: Vec<(usize, usize, String)> = + document.nodes.iter().filter_map(scalar_mark).collect(); + found.sort(); + found + }) + .map_err(|error| error.kind) +} + +fn scalar_mark(node: &ComposedYamlNode) -> Option<(usize, usize, String)> { + node.content + .as_scalar() + .map(|value| (node.start, node.end, value.to_owned())) +} + +pub(super) fn owned_marks( + marks: Result<&[(usize, usize, &str)], ConfigErrorKind>, +) -> Result, ConfigErrorKind> { + marks.map(|items| { + items + .iter() + .map(|(start, end, value)| (*start, *end, (*value).to_owned())) + .collect() + }) +} diff --git a/crates/sqlbuild-config/src/yaml/tests/test_compose_marks.rs b/crates/sqlbuild-config/src/yaml/tests/test_compose_marks.rs new file mode 100644 index 0000000000..f65c0236d3 --- /dev/null +++ b/crates/sqlbuild-config/src/yaml/tests/test_compose_marks.rs @@ -0,0 +1,120 @@ +use crate::yaml::tests::helpers::{owned_marks, scalar_marks}; +use crate::yaml::tests::test_types::ComposeMarksTestCase; + +#[test] +fn given_yaml_documents_when_composing_then_scalar_marks_match_pyyaml() { + let test_cases = [ + ComposeMarksTestCase { + description: "quoted and plain scalars start at their token and end after it", + text: "a: 'single ''quoted'' value'\nb: plain value \nc: \"multi\n line\"\n", + expected_scalars: Ok(&[ + (0, 1, "a"), + (3, 28, "single 'quoted' value"), + (29, 30, "b"), + (32, 43, "plain value"), + (47, 48, "c"), + (50, 64, "multi line"), + ]), + }, + ComposeMarksTestCase { + description: "block scalars start at their indicator and end after their last line break", + text: "d: |\n x __ref(\"y\")\n z\ne: >-\n folded\n\n\nf: g", + expected_scalars: Ok(&[ + (0, 1, "d"), + (3, 24, "x __ref(\"y\")\nz\n"), + (24, 25, "e"), + (27, 41, "folded"), + (41, 42, "f"), + (44, 45, "g"), + ]), + }, + ComposeMarksTestCase { + description: "a byte order mark counts in the marks, as PyYAML's reader keeps it", + text: "\u{feff}k: v", + expected_scalars: Ok(&[(1, 2, "k"), (4, 5, "v")]), + }, + ComposeMarksTestCase { + description: "an anchored scalar starts at its anchor and its alias is the same node", + text: "a: &x v\nb: *x\n", + expected_scalars: Ok(&[(0, 1, "a"), (3, 7, "v"), (8, 9, "b")]), + }, + ComposeMarksTestCase { + description: "a scalar with a tag and an anchor starts at its first property", + text: "a: !!str 12\nb: !!str &y q\nc: &t !!str 'x'\n", + expected_scalars: Ok(&[ + (0, 1, "a"), + (3, 11, "12"), + (12, 13, "b"), + (15, 25, "q"), + (26, 27, "c"), + (29, 41, "x"), + ]), + }, + ComposeMarksTestCase { + description: "a verbatim tag with a comma and an anchored block scalar", + text: "x: ! v\nc: &c |\n txt\n", + expected_scalars: Ok(&[(0, 1, "x"), (3, 29, "v"), (30, 31, "c"), (33, 44, "txt\n")]), + }, + ComposeMarksTestCase { + description: "anchors inside flow and block collections", + text: "k: [&e a, *e]\nm: &n\n - &o x\n", + expected_scalars: Ok(&[(0, 1, "k"), (4, 8, "a"), (14, 15, "m"), (24, 28, "x")]), + }, + ComposeMarksTestCase { + description: "a tag after a double-quoted value holding ' # ' starts at the tag", + text: "sources: [{name: raw_orders, description: \"Orders # archived\", expression: !!str \"(SELECT 1 AS id)\"}]\n", + expected_scalars: Ok(&[ + (0, 7, "sources"), + (11, 15, "name"), + (17, 27, "raw_orders"), + (29, 40, "description"), + (42, 61, "Orders # archived"), + (63, 73, "expression"), + (75, 99, "(SELECT 1 AS id)"), + ]), + }, + ComposeMarksTestCase { + description: "a tag after a single-quoted value holding ' # ' starts at the tag", + text: "sources: [{name: raw_orders, description: 'Orders # archived', expression: !!str '(SELECT 1 AS id)'}]\n", + expected_scalars: Ok(&[ + (0, 7, "sources"), + (11, 15, "name"), + (17, 27, "raw_orders"), + (29, 40, "description"), + (42, 61, "Orders # archived"), + (63, 73, "expression"), + (75, 99, "(SELECT 1 AS id)"), + ]), + }, + ComposeMarksTestCase { + description: "real comments between and after properties are skipped", + text: "k: &a # real comment\n v\nq: \"x # y\" # c\nr: !!str &b 'z' # c\n", + expected_scalars: Ok(&[ + (0, 1, "k"), + (3, 25, "v"), + (26, 27, "q"), + (29, 36, "x # y"), + (42, 43, "r"), + (45, 57, "z"), + ]), + }, + ComposeMarksTestCase { + description: "a quoted scalar ends at its closing quote, before spaces and a comment", + text: "k: 'it''s' # 'x'\nd: \"a \\\" b\" \n", + expected_scalars: Ok(&[ + (0, 1, "k"), + (3, 10, "it's"), + (19, 20, "d"), + (22, 30, "a \" b"), + ]), + }, + ]; + for test_case in test_cases { + assert_eq!( + scalar_marks(test_case.text), + owned_marks(test_case.expected_scalars), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/yaml/tests/test_types.rs b/crates/sqlbuild-config/src/yaml/tests/test_types.rs index 8e36a9ebb0..e447fb6378 100644 --- a/crates/sqlbuild-config/src/yaml/tests/test_types.rs +++ b/crates/sqlbuild-config/src/yaml/tests/test_types.rs @@ -12,3 +12,10 @@ pub(super) struct HostileYamlTestCase { pub(super) text: String, pub(super) expected_error: ConfigErrorKind, } + +pub(super) struct ComposeMarksTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + /// `(start, end, value)` of every scalar, sorted, as PyYAML 6 marks them. + pub(super) expected_scalars: Result<&'static [(usize, usize, &'static str)], ConfigErrorKind>, +} diff --git a/crates/sqlbuild-core/src/json/_helpers/emitter.rs b/crates/sqlbuild-core/src/json/_helpers/emitter.rs index 809c66b1d5..3a275b282b 100644 --- a/crates/sqlbuild-core/src/json/_helpers/emitter.rs +++ b/crates/sqlbuild-core/src/json/_helpers/emitter.rs @@ -1,9 +1,10 @@ //! Serialize a [`JsonValue`] in the layout of one Python serializer. use crate::json::_helpers::floats::{orjson_text, python_repr}; -use crate::json::_helpers::strings::{StringEscaping, json_string}; +use crate::json::_helpers::strings::{StringEscaping, json_string, json_utf16_string}; use crate::json::constants::{ - MAX_NESTING_DEPTH, ORJSON_MAX_INTEGER, ORJSON_MIN_INTEGER, PYTHON_INT_MAX_STR_DIGITS, + ORJSON_MAX_CONTAINER_DEPTH, ORJSON_MAX_INTEGER, ORJSON_MIN_INTEGER, PYTHON_INT_MAX_STR_DIGITS, + STDLIB_MAX_CONTAINER_DEPTH, }; use crate::json::errors::JsonEmitError; use crate::json::models::{JsonDialect, JsonInteger, JsonValue}; @@ -78,8 +79,23 @@ impl<'options> Layout<'options> { } } + /// Whether the serializer refuses this container at `level`; orjson skips empty lists. + fn too_deep(&self, value: &JsonValue, level: usize) -> bool { + let counted: bool = match value { + JsonValue::Object(_) => true, + JsonValue::Array(items) => !self.orjson || !items.is_empty(), + _ => false, + }; + let limit: usize = if self.orjson { + ORJSON_MAX_CONTAINER_DEPTH + } else { + STDLIB_MAX_CONTAINER_DEPTH + }; + counted && level >= limit + } + fn value(&self, value: &JsonValue, level: usize) -> Result { - if level > MAX_NESTING_DEPTH { + if self.too_deep(value, level) { return Err(JsonEmitError::NestingTooDeep); } match value { @@ -88,6 +104,7 @@ impl<'options> Layout<'options> { JsonValue::Integer(number) => self.integer(number), JsonValue::Float(number) => self.float(*number), JsonValue::String(text) => Ok(json_string(text, self.escaping)), + JsonValue::Utf16(units) => json_utf16_string(units, self.escaping), JsonValue::Array(items) => { let members = items .iter() diff --git a/crates/sqlbuild-core/src/json/_helpers/strings.rs b/crates/sqlbuild-core/src/json/_helpers/strings.rs index 169c27da2e..9152d34258 100644 --- a/crates/sqlbuild-core/src/json/_helpers/strings.rs +++ b/crates/sqlbuild-core/src/json/_helpers/strings.rs @@ -1,6 +1,7 @@ //! JSON string escaping of `json.dumps` and orjson. use crate::json::constants::FIRST_PRINTABLE_ASCII; +use crate::json::errors::JsonEmitError; /// How a serializer escapes characters outside printable ASCII. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -48,3 +49,30 @@ pub(crate) fn json_string(text: &str, escaping: StringEscaping) -> String { output.push('"'); output } + +/// UTF-16 `units` as a quoted JSON string; `ensure_ascii` escapes a lone surrogate as `\uXXXX`. +pub(crate) fn json_utf16_string( + units: &[u16], + escaping: StringEscaping, +) -> Result { + let mut output = String::with_capacity(units.len() + 2); + output.push('"'); + for decoded in char::decode_utf16(units.iter().copied()) { + match (decoded, escaping) { + (Ok(character), _) => output.push_str(&escaped_or_plain(character, escaping)), + (Err(lone), StringEscaping::Ascii) => { + output.push_str(&unicode_escape(u32::from(lone.unpaired_surrogate()))); + } + (Err(_), StringEscaping::Unicode) => return Err(JsonEmitError::LoneSurrogate), + } + } + output.push('"'); + Ok(output) +} + +fn escaped_or_plain(character: char, escaping: StringEscaping) -> String { + match character { + ' '..='~' if character != '"' && character != '\\' => character.to_string(), + _ => escaped(character, escaping), + } +} diff --git a/crates/sqlbuild-core/src/json/constants.rs b/crates/sqlbuild-core/src/json/constants.rs index fab04a225b..cee6bce4e7 100644 --- a/crates/sqlbuild-core/src/json/constants.rs +++ b/crates/sqlbuild-core/src/json/constants.rs @@ -9,5 +9,7 @@ pub(crate) const PYTHON_EXPONENT_DIGITS: usize = 2; pub(crate) const ORJSON_EXPONENT_DIGITS: usize = 1; /// CPython's default `sys.get_int_max_str_digits()`, beyond which `int.__repr__` raises. pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4_300; -/// Containers nested deeper than this are refused; orjson's own limit is 255 and json's is higher. -pub(crate) const MAX_NESTING_DEPTH: usize = 128; +/// orjson refuses a dict, or a non-empty list, nested this many containers deep. +pub(crate) const ORJSON_MAX_CONTAINER_DEPTH: usize = 254; +/// CPython's default recursion limit, which bounds how deep `json.dumps` can nest. +pub(crate) const STDLIB_MAX_CONTAINER_DEPTH: usize = 1_000; diff --git a/crates/sqlbuild-core/src/json/errors.rs b/crates/sqlbuild-core/src/json/errors.rs index 0c731dec9a..bf447b5575 100644 --- a/crates/sqlbuild-core/src/json/errors.rs +++ b/crates/sqlbuild-core/src/json/errors.rs @@ -9,6 +9,8 @@ pub enum JsonEmitError { IntegerOutOfRange, /// `json.dumps` met an integer with more digits than Python's string conversion limit. IntegerTooLong, - /// The value nests deeper than the native emitter writes; callers fall back to Python. + /// orjson's recursion limit, or `json.dumps` beyond Python's default recursion limit. NestingTooDeep, + /// A lone surrogate under an encoder that writes text unescaped; Python cannot encode it. + LoneSurrogate, } diff --git a/crates/sqlbuild-core/src/json/main/mod.rs b/crates/sqlbuild-core/src/json/main/mod.rs index ca5e541504..c295400d51 100644 --- a/crates/sqlbuild-core/src/json/main/mod.rs +++ b/crates/sqlbuild-core/src/json/main/mod.rs @@ -1 +1,2 @@ pub mod dumps; +pub mod python_float_repr; diff --git a/crates/sqlbuild-core/src/json/main/python_float_repr.rs b/crates/sqlbuild-core/src/json/main/python_float_repr.rs new file mode 100644 index 0000000000..41d08736d5 --- /dev/null +++ b/crates/sqlbuild-core/src/json/main/python_float_repr.rs @@ -0,0 +1,6 @@ +//! Python's `repr(float)`. + +/// `repr(value)`, including `nan`, `inf` and `-inf`. +pub fn python_float_repr(value: f64) -> String { + crate::json::_helpers::floats::python_repr(value) +} diff --git a/crates/sqlbuild-core/src/json/models.rs b/crates/sqlbuild-core/src/json/models.rs index 351ec48f0a..83081faf87 100644 --- a/crates/sqlbuild-core/src/json/models.rs +++ b/crates/sqlbuild-core/src/json/models.rs @@ -8,6 +8,8 @@ pub enum JsonValue { Integer(JsonInteger), Float(f64), String(String), + /// A Python `str` as UTF-16 code units, for text holding lone surrogates. + Utf16(Vec), Array(Vec), Object(Vec<(String, JsonValue)>), } diff --git a/crates/sqlbuild-core/src/json/tests/helpers.rs b/crates/sqlbuild-core/src/json/tests/helpers.rs index 25229c37ff..b7fc4824ff 100644 --- a/crates/sqlbuild-core/src/json/tests/helpers.rs +++ b/crates/sqlbuild-core/src/json/tests/helpers.rs @@ -55,3 +55,27 @@ pub(super) fn nested_arrays(depth: usize) -> JsonValue { JsonValue::Array(vec![inner]) }) } + +/// `"q"\` then a lone low surrogate, a non-BMP pair, and a lone high surrogate, as UTF-16. +pub(super) fn lone_surrogate_text() -> JsonValue { + JsonValue::Utf16(vec![ + 0x0022, 0x0071, 0x0022, 0x005c, 0x00e9, 0xdcff, 0xd83d, 0xde00, 0xd83d, + ]) +} + +/// `inner` wrapped in `depth` single-item lists. +pub(super) fn wrapped_in_arrays(depth: usize, inner: JsonValue) -> JsonValue { + (0..depth).fold(inner, |value, _| JsonValue::Array(vec![value])) +} + +/// Run `check` on a thread with room for Python-depth recursion. +pub(super) fn on_large_stack(check: impl FnOnce() -> bool + Send + 'static) -> bool { + std::thread::Builder::new() + .stack_size(LARGE_STACK_BYTES) + .spawn(check) + .expect("test thread") + .join() + .expect("test thread result") +} + +const LARGE_STACK_BYTES: usize = 256 * 1024 * 1024; diff --git a/crates/sqlbuild-core/src/json/tests/test_dumps.rs b/crates/sqlbuild-core/src/json/tests/test_dumps.rs index 2d1584eb5d..d08a51fe24 100644 --- a/crates/sqlbuild-core/src/json/tests/test_dumps.rs +++ b/crates/sqlbuild-core/src/json/tests/test_dumps.rs @@ -1,9 +1,14 @@ use crate::json::_helpers::floats::python_repr; use crate::json::errors::JsonEmitError; use crate::json::main::dumps::dumps; -use crate::json::models::{JsonDialect, JsonInteger, OrjsonOptions, StdlibJsonOptions}; -use crate::json::tests::helpers::{digits, nested_arrays, non_finite_floats, sample_document}; -use crate::json::tests::test_types::{DumpsTestCase, FloatReprTestCase, IntegerParseTestCase}; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue, OrjsonOptions, StdlibJsonOptions}; +use crate::json::tests::helpers::{ + digits, lone_surrogate_text, nested_arrays, non_finite_floats, on_large_stack, sample_document, + wrapped_in_arrays, +}; +use crate::json::tests::test_types::{ + DumpsTestCase, FloatReprTestCase, IntegerParseTestCase, NestingTestCase, +}; const BIG: &str = "1180591620717411303424"; const U64_MAX: &str = "18446744073709551615"; @@ -108,16 +113,25 @@ fn given_python_serializer_options_when_dumping_then_output_matches_python_bytes expected_text: Err(JsonEmitError::IntegerTooLong), }, DumpsTestCase { - description: "deeply nested values are refused instead of exhausting the stack", - value: nested_arrays(1_000), + description: "json.dumps escapes lone surrogates beside escapes and a non-BMP pair", + value: lone_surrogate_text(), dialect: JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), - expected_text: Err(JsonEmitError::NestingTooDeep), + expected_text: Ok(r#""\"q\"\\\u00e9\udcff\ud83d\ude00\ud83d""#), + }, + DumpsTestCase { + description: "json.dumps ensure_ascii=False cannot write a lone surrogate as text", + value: lone_surrogate_text(), + dialect: JsonDialect::Stdlib(StdlibJsonOptions { + ensure_ascii: false, + ..StdlibJsonOptions::new(None) + }), + expected_text: Err(JsonEmitError::LoneSurrogate), }, DumpsTestCase { - description: "orjson refuses deeply nested values too", - value: nested_arrays(130), + description: "orjson refuses lone surrogates", + value: lone_surrogate_text(), dialect: JsonDialect::Orjson(OrjsonOptions::default()), - expected_text: Err(JsonEmitError::NestingTooDeep), + expected_text: Err(JsonEmitError::LoneSurrogate), }, DumpsTestCase { description: "nesting within the limit is written", @@ -265,3 +279,62 @@ fn given_python_integer_text_when_parsing_then_text_is_canonical() { ); } } + +#[test] +fn given_nesting_near_python_limits_when_dumping_then_refusal_matches_python() { + let orjson = || JsonDialect::Orjson(OrjsonOptions::default()); + let stdlib = || JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))); + let test_cases = [ + NestingTestCase { + description: "orjson writes 254 non-empty lists", + value: wrapped_in_arrays(254, JsonValue::Bool(true)), + dialect: orjson(), + expected_written: true, + }, + NestingTestCase { + description: "orjson refuses the 255th non-empty list", + value: wrapped_in_arrays(255, JsonValue::Bool(true)), + dialect: orjson(), + expected_written: false, + }, + NestingTestCase { + description: "orjson does not count an empty list", + value: nested_arrays(255), + dialect: orjson(), + expected_written: true, + }, + NestingTestCase { + description: "orjson counts an empty dict", + value: wrapped_in_arrays(254, JsonValue::Object(Vec::new())), + dialect: orjson(), + expected_written: false, + }, + NestingTestCase { + description: "orjson writes an empty dict one level up", + value: wrapped_in_arrays(253, JsonValue::Object(Vec::new())), + dialect: orjson(), + expected_written: true, + }, + NestingTestCase { + description: "json.dumps writes what orjson refuses, up to the recursion limit", + value: nested_arrays(1_000), + dialect: stdlib(), + expected_written: true, + }, + NestingTestCase { + description: "json.dumps refuses nesting beyond Python's default recursion limit", + value: nested_arrays(1_001), + dialect: stdlib(), + expected_written: false, + }, + ]; + for test_case in test_cases { + let written: bool = + on_large_stack(move || dumps(&test_case.value, &test_case.dialect).is_ok()); + assert_eq!( + written, test_case.expected_written, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/json/tests/test_types.rs b/crates/sqlbuild-core/src/json/tests/test_types.rs index 2ce319a3b0..b93ef8fc10 100644 --- a/crates/sqlbuild-core/src/json/tests/test_types.rs +++ b/crates/sqlbuild-core/src/json/tests/test_types.rs @@ -19,3 +19,10 @@ pub(super) struct IntegerParseTestCase { pub(super) text: &'static str, pub(super) expected_decimal: Option<&'static str>, } + +pub(super) struct NestingTestCase { + pub(super) description: &'static str, + pub(super) value: JsonValue, + pub(super) dialect: JsonDialect, + pub(super) expected_written: bool, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index d94e5102aa..10e15f1ed3 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -6,6 +6,7 @@ pub mod constants; pub mod json; pub mod models; pub mod panics; +pub mod posix_path; pub mod text; #[cfg(test)] diff --git a/crates/sqlbuild-core/src/posix_path/constants.rs b/crates/sqlbuild-core/src/posix_path/constants.rs new file mode 100644 index 0000000000..35e9615633 --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/constants.rs @@ -0,0 +1,10 @@ +//! `PurePosixPath` separators and anchors. + +pub(crate) const SEPARATOR: char = '/'; +pub(crate) const ROOT: &str = "/"; +/// POSIX keeps exactly two leading slashes as their own root. +pub(crate) const DOUBLE_ROOT: &str = "//"; +pub(crate) const CURRENT: &str = "."; +pub(crate) const SUFFIX_MARK: char = '.'; +/// Python 3.14 reads the suffix as `os.path.splitext`; earlier releases need text after the dot. +pub(crate) const SPLITEXT_SUFFIX_MINOR: u8 = 14; diff --git a/crates/sqlbuild-core/src/posix_path/main/mod.rs b/crates/sqlbuild-core/src/posix_path/main/mod.rs new file mode 100644 index 0000000000..e4ab185549 --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/main/mod.rs @@ -0,0 +1,5 @@ +pub mod posix_join; +pub mod posix_name; +pub mod posix_parent; +pub mod posix_parts; +pub mod posix_suffix; diff --git a/crates/sqlbuild-core/src/posix_path/main/posix_join.rs b/crates/sqlbuild-core/src/posix_path/main/posix_join.rs new file mode 100644 index 0000000000..bd84e8bf8f --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/main/posix_join.rs @@ -0,0 +1,15 @@ +//! `PurePosixPath(*parts).as_posix()` of parts `posix_parts` returned. + +use crate::posix_path::constants::{CURRENT, DOUBLE_ROOT, ROOT}; + +/// The parts joined by `/` after their root, or `.` when there are none. +#[must_use] +pub fn posix_join(parts: &[&str]) -> String { + match parts.split_first() { + None => CURRENT.to_owned(), + Some((root, rest)) if *root == ROOT || *root == DOUBLE_ROOT => { + format!("{root}{}", rest.join(ROOT)) + } + Some(_) => parts.join(ROOT), + } +} diff --git a/crates/sqlbuild-core/src/posix_path/main/posix_name.rs b/crates/sqlbuild-core/src/posix_path/main/posix_name.rs new file mode 100644 index 0000000000..257e69050e --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/main/posix_name.rs @@ -0,0 +1,13 @@ +//! `PurePosixPath(path).name`. + +use crate::posix_path::constants::{DOUBLE_ROOT, ROOT}; +use crate::posix_path::main::posix_parts::posix_parts; + +/// The last component, or an empty name for a bare root or `.`. +#[must_use] +pub fn posix_name(path: &str) -> &str { + match posix_parts(path).last() { + Some(last) if *last != ROOT && *last != DOUBLE_ROOT => last, + _ => "", + } +} diff --git a/crates/sqlbuild-core/src/posix_path/main/posix_parent.rs b/crates/sqlbuild-core/src/posix_path/main/posix_parent.rs new file mode 100644 index 0000000000..ebac4ad53c --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/main/posix_parent.rs @@ -0,0 +1,16 @@ +//! `PurePosixPath(path).parent.as_posix()`. + +use crate::posix_path::constants::{DOUBLE_ROOT, ROOT}; +use crate::posix_path::main::posix_join::posix_join; +use crate::posix_path::main::posix_parts::posix_parts; + +/// The path without its last component; a bare root or `.` is its own parent. +#[must_use] +pub fn posix_parent(path: &str) -> String { + let parts: Vec<&str> = posix_parts(path); + match parts.as_slice() { + [root] if *root == ROOT || *root == DOUBLE_ROOT => (*root).to_owned(), + [] => posix_join(&[]), + [init @ .., _] => posix_join(init), + } +} diff --git a/crates/sqlbuild-core/src/posix_path/main/posix_parts.rs b/crates/sqlbuild-core/src/posix_path/main/posix_parts.rs new file mode 100644 index 0000000000..35afa5ac35 --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/main/posix_parts.rs @@ -0,0 +1,19 @@ +//! `PurePosixPath(path).parts`. + +use crate::posix_path::constants::{CURRENT, DOUBLE_ROOT, ROOT, SEPARATOR}; + +/// The root, if any, then every component except empty and `.` ones. +#[must_use] +pub fn posix_parts(path: &str) -> Vec<&str> { + let mut parts: Vec<&str> = Vec::new(); + if path.starts_with(DOUBLE_ROOT) && !path[DOUBLE_ROOT.len()..].starts_with(SEPARATOR) { + parts.push(DOUBLE_ROOT); + } else if path.starts_with(SEPARATOR) { + parts.push(ROOT); + } + parts.extend( + path.split(SEPARATOR) + .filter(|part| !part.is_empty() && *part != CURRENT), + ); + parts +} diff --git a/crates/sqlbuild-core/src/posix_path/main/posix_suffix.rs b/crates/sqlbuild-core/src/posix_path/main/posix_suffix.rs new file mode 100644 index 0000000000..6e795eac7f --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/main/posix_suffix.rs @@ -0,0 +1,21 @@ +//! `PurePosixPath(path).suffix` of one Python release. + +use crate::posix_path::constants::{SPLITEXT_SUFFIX_MINOR, SUFFIX_MARK}; +use crate::posix_path::main::posix_name::posix_name; +use crate::text::models::PythonText; + +/// The name's final `.`-led part, or an empty suffix. +#[must_use] +pub fn posix_suffix(python: PythonText, path: &str) -> &str { + let name: &str = posix_name(path); + let Some(dot) = name.rfind(SUFFIX_MARK) else { + return ""; + }; + let leading: &str = &name[..dot]; + let named: bool = if python.minor_version() >= SPLITEXT_SUFFIX_MINOR { + leading.chars().any(|character| character != SUFFIX_MARK) + } else { + !leading.is_empty() && dot + SUFFIX_MARK.len_utf8() < name.len() + }; + if named { &name[dot..] } else { "" } +} diff --git a/crates/sqlbuild-core/src/posix_path/mod.rs b/crates/sqlbuild-core/src/posix_path/mod.rs new file mode 100644 index 0000000000..9b926d1cd2 --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/posix_path/tests.rs b/crates/sqlbuild-core/src/posix_path/tests.rs new file mode 100644 index 0000000000..b3bcb924a0 --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_posix_path.rs"] +mod posix_path; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs new file mode 100644 index 0000000000..f47d6f7593 --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs @@ -0,0 +1,149 @@ +use crate::posix_path::main::posix_join::posix_join; +use crate::posix_path::main::posix_name::posix_name; +use crate::posix_path::main::posix_parent::posix_parent; +use crate::posix_path::main::posix_parts::posix_parts; +use crate::posix_path::main::posix_suffix::posix_suffix; +use crate::posix_path::tests::test_types::PosixPathTestCase; +use crate::text::main::python_text::python_text; +use crate::text::models::PythonText; + +#[test] +fn given_paths_when_reading_pure_posix_parts_then_they_match_pathlib() { + let test_cases = [ + PosixPathTestCase { + description: "a relative file path", + path: "models/a.sql", + expected_parts: &["models", "a.sql"], + expected_parent: "models", + expected_name: "a.sql", + expected_suffixes: (".sql", ".sql"), + }, + PosixPathTestCase { + description: "current, empty and trailing components are dropped", + path: "./a//b/", + expected_parts: &["a", "b"], + expected_parent: "a", + expected_name: "b", + expected_suffixes: ("", ""), + }, + PosixPathTestCase { + description: "an absolute path keeps its root", + path: "/abs/x.yml", + expected_parts: &["/", "abs", "x.yml"], + expected_parent: "/abs", + expected_name: "x.yml", + expected_suffixes: (".yml", ".yml"), + }, + PosixPathTestCase { + description: "exactly two leading slashes are their own root", + path: "//net/x", + expected_parts: &["//", "net", "x"], + expected_parent: "//net", + expected_name: "x", + expected_suffixes: ("", ""), + }, + PosixPathTestCase { + description: "three leading slashes collapse to one root", + path: "///three/x", + expected_parts: &["/", "three", "x"], + expected_parent: "/three", + expected_name: "x", + expected_suffixes: ("", ""), + }, + PosixPathTestCase { + description: "the current directory has no parts", + path: ".", + expected_parts: &[], + expected_parent: ".", + expected_name: "", + expected_suffixes: ("", ""), + }, + PosixPathTestCase { + description: "empty text is the current directory", + path: "", + expected_parts: &[], + expected_parent: ".", + expected_name: "", + expected_suffixes: ("", ""), + }, + PosixPathTestCase { + description: "a parent component is kept", + path: "a/..", + expected_parts: &["a", ".."], + expected_parent: "a", + expected_name: "..", + expected_suffixes: ("", ""), + }, + PosixPathTestCase { + description: "a leading-dot name has a suffix only before Python 3.14", + path: "..a", + expected_parts: &["..a"], + expected_parent: ".", + expected_name: "..a", + expected_suffixes: (".a", ""), + }, + PosixPathTestCase { + description: "a trailing dot is a suffix only from Python 3.14", + path: "d/a.", + expected_parts: &["d", "a."], + expected_parent: "d", + expected_name: "a.", + expected_suffixes: ("", "."), + }, + PosixPathTestCase { + description: "the last dot starts the suffix", + path: "d/...b.c", + expected_parts: &["d", "...b.c"], + expected_parent: "d", + expected_name: "...b.c", + expected_suffixes: (".c", ".c"), + }, + PosixPathTestCase { + description: "non-ASCII components are whole parts", + path: "caf\u{e9}/\u{1F600}.sql", + expected_parts: &["caf\u{e9}", "\u{1F600}.sql"], + expected_parent: "caf\u{e9}", + expected_name: "\u{1F600}.sql", + expected_suffixes: (".sql", ".sql"), + }, + PosixPathTestCase { + description: "a bare root is its own parent", + path: "/", + expected_parts: &["/"], + expected_parent: "/", + expected_name: "", + expected_suffixes: ("", ""), + }, + ]; + let python_312: PythonText = python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"); + let python_314: PythonText = python_text((3, 14), "16.0.0").expect("Python 3.14 is supported"); + for test_case in test_cases { + let parts: Vec<&str> = posix_parts(test_case.path); + + assert_eq!( + ( + parts.as_slice(), + posix_parent(test_case.path), + posix_name(test_case.path), + ( + posix_suffix(python_312, test_case.path), + posix_suffix(python_314, test_case.path) + ), + ), + ( + test_case.expected_parts, + test_case.expected_parent.to_owned(), + test_case.expected_name, + test_case.expected_suffixes, + ), + "{}", + test_case.description + ); + assert_eq!( + posix_parts(&posix_join(&parts)), + parts, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_types.rs b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs new file mode 100644 index 0000000000..8478707013 --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs @@ -0,0 +1,9 @@ +pub(super) struct PosixPathTestCase { + pub(super) description: &'static str, + pub(super) path: &'static str, + pub(super) expected_parts: &'static [&'static str], + pub(super) expected_parent: &'static str, + pub(super) expected_name: &'static str, + /// `suffix` under Python 3.12 and under Python 3.14. + pub(super) expected_suffixes: (&'static str, &'static str), +} diff --git a/crates/sqlbuild-core/src/text/_helpers/active_text.rs b/crates/sqlbuild-core/src/text/_helpers/active_text.rs index a907ea865f..28effa3d58 100644 --- a/crates/sqlbuild-core/src/text/_helpers/active_text.rs +++ b/crates/sqlbuild-core/src/text/_helpers/active_text.rs @@ -2,7 +2,9 @@ use std::sync::OnceLock; -use crate::text::_helpers::alnum_tables::{ALNUM_TABLES, ALPHA_TABLES, DECIMAL_TABLES}; +use crate::text::_helpers::alnum_tables::{ + ALNUM_TABLES, ALPHA_TABLES, DECIMAL_TABLES, PRINTABLE_TABLES, +}; use crate::text::_helpers::case_tables::{CASEFOLD_TABLES, IGNORECASE_KEY_TABLES, UPPER_TABLES}; use crate::text::models::{CleandocMargin, PythonText}; @@ -19,6 +21,7 @@ pub(crate) fn active() -> PythonText { alnum_ranges: ALNUM_TABLES[0].1, alpha_ranges: ALPHA_TABLES[0].1, decimal_ranges: DECIMAL_TABLES[0].1, + printable_ranges: PRINTABLE_TABLES[0].1, casefold_mappings: CASEFOLD_TABLES[0].1, upper_mappings: UPPER_TABLES[0].1, ignorecase_keys: IGNORECASE_KEY_TABLES[0].1, diff --git a/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs index 4c81042156..e889b3c4ba 100644 --- a/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs +++ b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs @@ -1,4 +1,4 @@ -//! The generated `str.isalnum()`, `str.isalpha()` and `str.isdecimal()` tables, per Unicode version. +//! The generated `str.isalnum()`, `isalpha()`, `isdecimal()` and `isprintable()` tables. /// `(unicodedata.unidata_version, ranges)` for every table native text helpers can select. pub(crate) const ALNUM_TABLES: [(&str, &[(u32, u32)]); 3] = [ @@ -20,3 +20,10 @@ pub(crate) const DECIMAL_TABLES: [(&str, &[(u32, u32)]); 3] = [ ("15.1.0", include!("../decimal_ranges/unicode_15_1_0.in")), ("16.0.0", include!("../decimal_ranges/unicode_16_0_0.in")), ]; + +/// `(unicodedata.unidata_version, ranges)` of `str.isprintable()`, in the same order. +pub(crate) const PRINTABLE_TABLES: [(&str, &[(u32, u32)]); 3] = [ + ("15.0.0", include!("../printable_ranges/unicode_15_0_0.in")), + ("15.1.0", include!("../printable_ranges/unicode_15_1_0.in")), + ("16.0.0", include!("../printable_ranges/unicode_16_0_0.in")), +]; diff --git a/crates/sqlbuild-core/src/text/_helpers/fnmatch.rs b/crates/sqlbuild-core/src/text/_helpers/fnmatch.rs new file mode 100644 index 0000000000..1351a7f5c7 --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/fnmatch.rs @@ -0,0 +1,194 @@ +//! `fnmatch._translate` pieces, read the way `re` reads the translated character sets. + +/// One translated pattern piece; every piece except `Star` matches exactly one character. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum Piece { + Star, + Any, + Literal(char), + Set { + negated: bool, + members: Vec<(char, char)>, + }, + /// Python's `(?!)` for a set whose every range was empty. + Never, +} + +impl Piece { + /// Whether this one-character piece matches the code point `character`, surrogates included. + pub(crate) fn matches(&self, character: u32) -> bool { + match self { + Self::Star | Self::Any => true, + Self::Literal(literal) => u32::from(*literal) == character, + Self::Set { negated, members } => { + let member: bool = members.iter().any(|(low, high)| { + u32::from(*low) <= character && character <= u32::from(*high) + }); + member != *negated + } + Self::Never => false, + } + } +} + +/// One translated set member: a literal character or the hyphen joining a range. +#[derive(Clone, Copy, PartialEq, Eq)] +enum SetAtom { + Character(char), + RangeHyphen, +} + +/// The pieces of `pattern`, with consecutive stars collapsed. +pub(crate) fn translate(pattern: &[char]) -> Vec { + let mut pieces: Vec = Vec::new(); + let mut index: usize = 0; + while index < pattern.len() { + let character: char = pattern[index]; + index += 1; + match character { + '*' if pieces.last() == Some(&Piece::Star) => {} + '*' => pieces.push(Piece::Star), + '?' => pieces.push(Piece::Any), + '[' => { + let (piece, next) = set(pattern, index); + pieces.push(piece); + index = next; + } + _ => pieces.push(Piece::Literal(character)), + } + } + pieces +} + +/// The set starting after `[` at `start`, and the index after it; an unclosed `[` is literal. +fn set(pattern: &[char], start: usize) -> (Piece, usize) { + let length: usize = pattern.len(); + let mut end: usize = start; + if end < length && pattern[end] == '!' { + end += 1; + } + if end < length && pattern[end] == ']' { + end += 1; + } + while end < length && pattern[end] != ']' { + end += 1; + } + if end >= length { + return (Piece::Literal('['), start); + } + let chunks: Vec> = if pattern[start..end].contains(&'-') { + hyphen_chunks(pattern, start, end) + } else { + vec![pattern[start..end].to_vec()] + }; + (set_piece(&chunks), end + 1) +} + +/// Python's chunks of a set holding hyphens, after it drops empty ranges. +fn hyphen_chunks(pattern: &[char], start: usize, end: usize) -> Vec> { + let mut chunks: Vec> = Vec::new(); + let mut chunk_start: usize = start; + let mut search: usize = if pattern[start] == '!' { + start + 2 + } else { + start + 1 + }; + while let Some(found) = (search..end).find(|&position| pattern[position] == '-') { + chunks.push(pattern[chunk_start..found].to_vec()); + chunk_start = found + 1; + search = found + 3; + } + let last: Vec = pattern[chunk_start..end].to_vec(); + match (last.is_empty(), chunks.last_mut()) { + (true, Some(previous)) => previous.push('-'), + _ => chunks.push(last), + } + let mut position: usize = chunks.len().saturating_sub(1); + while position > 0 { + let (left, right) = chunks.split_at_mut(position); + let previous: &mut Vec = &mut left[position - 1]; + if let (Some(&high), Some(&low)) = (previous.last(), right[0].first()) + && high > low + { + previous.pop(); + previous.extend(right[0].iter().skip(1)); + chunks.remove(position); + } + position -= 1; + } + chunks +} + +/// The piece `re` reads from the chunks joined by range hyphens. +fn set_piece(chunks: &[Vec]) -> Piece { + let mut atoms: Vec = Vec::new(); + for (position, chunk) in chunks.iter().enumerate() { + if position > 0 { + atoms.push(SetAtom::RangeHyphen); + } + atoms.extend(chunk.iter().map(|character| SetAtom::Character(*character))); + } + match atoms.as_slice() { + [] => return Piece::Never, + [SetAtom::Character('!')] => return Piece::Any, + _ => {} + } + let negated: bool = atoms[0] == SetAtom::Character('!'); + let mut members: Vec<(char, char)> = Vec::new(); + let mut index: usize = usize::from(negated); + while index < atoms.len() { + let low: char = atom_character(atoms[index]); + match (atoms.get(index + 1), atoms.get(index + 2)) { + (Some(SetAtom::RangeHyphen), Some(high)) => { + members.push((low, atom_character(*high))); + index += 3; + } + _ => { + members.push((low, low)); + index += 1; + } + } + } + Piece::Set { negated, members } +} + +/// A set atom's character; a hyphen `re` cannot join into a range is literal. +fn atom_character(atom: SetAtom) -> char { + match atom { + SetAtom::Character(character) => character, + SetAtom::RangeHyphen => '-', + } +} + +/// Whether the code points of `name` match `pattern`, as `re.match` of `fnmatch.translate` does. +pub(crate) fn code_points_match(name: &[u32], pattern: &str) -> bool { + let pieces: Vec = translate(&pattern.chars().collect::>()); + if pieces.contains(&Piece::Never) { + return false; + } + let (mut piece, mut character) = (0_usize, 0_usize); + let mut resume: Option<(usize, usize)> = None; + while character < name.len() { + match pieces.get(piece) { + Some(Piece::Star) => { + resume = Some((piece, character)); + piece += 1; + } + Some(single) if single.matches(name[character]) => { + piece += 1; + character += 1; + } + _ => match resume { + Some((star, consumed)) => { + piece = star + 1; + character = consumed + 1; + resume = Some((star, consumed + 1)); + } + None => return false, + }, + } + } + pieces[piece..] + .iter() + .all(|remaining| *remaining == Piece::Star) +} diff --git a/crates/sqlbuild-core/src/text/_helpers/mod.rs b/crates/sqlbuild-core/src/text/_helpers/mod.rs index 07c6ab70ae..310686ca09 100644 --- a/crates/sqlbuild-core/src/text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/text/_helpers/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod alnum_tables; pub(crate) mod case_tables; pub(crate) mod cleandoc; pub(crate) mod code_point_ranges; +pub(crate) mod fnmatch; pub(crate) mod sequence_matcher; diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index 94aabc44f6..ff4d674317 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -9,7 +9,10 @@ pub mod is_python_word; pub mod python_casefold; pub mod python_cleandoc; pub mod python_decimal_value; +pub mod python_fnmatchcase; +pub mod python_glob_name_match; pub mod python_ignorecase_key; +pub mod python_splitlines; pub mod python_str_repr; pub mod python_strip; pub mod python_text; diff --git a/crates/sqlbuild-core/src/text/main/python_fnmatchcase.rs b/crates/sqlbuild-core/src/text/main/python_fnmatchcase.rs new file mode 100644 index 0000000000..9c0e3d87c5 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_fnmatchcase.rs @@ -0,0 +1,10 @@ +//! Python's `fnmatch.fnmatchcase`. + +use crate::text::_helpers::fnmatch::code_points_match; + +/// Whether `name` matches the shell-style `pattern`, case-sensitively and across newlines. +#[must_use] +pub fn python_fnmatchcase(name: &str, pattern: &str) -> bool { + let code_points: Vec = name.chars().map(u32::from).collect(); + code_points_match(&code_points, pattern) +} diff --git a/crates/sqlbuild-core/src/text/main/python_glob_name_match.rs b/crates/sqlbuild-core/src/text/main/python_glob_name_match.rs new file mode 100644 index 0000000000..b9e00ab177 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_glob_name_match.rs @@ -0,0 +1,25 @@ +//! A `Path.glob` name match: `fnmatch.translate` compiled with or without `re.IGNORECASE`. + +use crate::text::_helpers::fnmatch::code_points_match; +use crate::text::main::active_python_text::active_python_text; +use crate::text::main::python_ignorecase_key::python_ignorecase_key; + +/// Whether the code points of `name`, lone surrogates included, match `pattern`. +#[must_use] +pub fn python_glob_name_match(name: &[u32], pattern: &str, ignore_case: bool) -> bool { + if !ignore_case { + return code_points_match(name, pattern); + } + let python = active_python_text(); + let key = |code_point: u32| { + char::from_u32(code_point).map_or(code_point, |character| { + u32::from(python_ignorecase_key(python, character)) + }) + }; + let folded_name: Vec = name.iter().map(|code_point| key(*code_point)).collect(); + let folded_pattern: String = pattern + .chars() + .map(|character| python_ignorecase_key(python, character)) + .collect(); + code_points_match(&folded_name, &folded_pattern) +} diff --git a/crates/sqlbuild-core/src/text/main/python_splitlines.rs b/crates/sqlbuild-core/src/text/main/python_splitlines.rs new file mode 100644 index 0000000000..fec51b7295 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_splitlines.rs @@ -0,0 +1,40 @@ +//! Python's `str.splitlines()` without `keepends`. + +/// Whether Python's `str.splitlines()` ends a line at `character`. +fn is_line_break(character: char) -> bool { + matches!( + character, + '\n' | '\r' + | '\u{b}' + | '\u{c}' + | '\u{1c}' + | '\u{1d}' + | '\u{1e}' + | '\u{85}' + | '\u{2028}' + | '\u{2029}' + ) +} + +/// The lines of `text`; `\r\n` is one break and a trailing break adds no empty line. +#[must_use] +pub fn python_splitlines(text: &str) -> Vec<&str> { + let mut lines: Vec<&str> = Vec::new(); + let mut start: usize = 0; + let mut characters = text.char_indices().peekable(); + while let Some((offset, character)) = characters.next() { + if !is_line_break(character) { + continue; + } + lines.push(&text[start..offset]); + start = offset + character.len_utf8(); + if character == '\r' && characters.peek().is_some_and(|(_, next)| *next == '\n') { + characters.next(); + start += 1; + } + } + if start < text.len() { + lines.push(&text[start..]); + } + lines +} diff --git a/crates/sqlbuild-core/src/text/main/python_str_repr.rs b/crates/sqlbuild-core/src/text/main/python_str_repr.rs index 092f9de9ed..4d0f7e70c0 100644 --- a/crates/sqlbuild-core/src/text/main/python_str_repr.rs +++ b/crates/sqlbuild-core/src/text/main/python_str_repr.rs @@ -1,18 +1,21 @@ //! Python's `repr` of a `str`. -/// Python's `repr` of a string, or `None` when it holds text whose escaping needs Unicode tables. -pub fn python_str_repr(value: &str) -> Option { - if !value.is_ascii() { - return None; - } +use crate::text::_helpers::code_point_ranges::in_ranges; +use crate::text::models::PythonText; + +/// Python's `repr` of a string under one Python's Unicode tables. +#[must_use] +pub fn python_str_repr(python: PythonText, value: &str) -> String { let quote: char = if value.contains('\'') && !value.contains('"') { '"' } else { '\'' }; + let printable: &[(u32, u32)] = python.printable_ranges; let mut text: String = String::with_capacity(value.len() + 2); text.push(quote); for character in value.chars() { + let code: u32 = u32::from(character); match character { '\\' => text.push_str("\\\\"), '\t' => text.push_str("\\t"), @@ -22,12 +25,13 @@ pub fn python_str_repr(value: &str) -> Option { text.push('\\'); text.push(character); } - _ if character < ' ' || character == '\x7f' => { - text.push_str(&format!("\\x{:02x}", u32::from(character))); - } - _ => text.push(character), + '\0'..='\x1f' | '\x7f' => text.push_str(&format!("\\x{code:02x}")), + _ if character.is_ascii() || in_ranges(printable, character) => text.push(character), + '\u{80}'..='\u{ff}' => text.push_str(&format!("\\x{code:02x}")), + '\u{100}'..='\u{ffff}' => text.push_str(&format!("\\u{code:04x}")), + _ => text.push_str(&format!("\\U{code:08x}")), } } text.push(quote); - Some(text) + text } diff --git a/crates/sqlbuild-core/src/text/main/python_text.rs b/crates/sqlbuild-core/src/text/main/python_text.rs index 2850a6d757..203c36d762 100644 --- a/crates/sqlbuild-core/src/text/main/python_text.rs +++ b/crates/sqlbuild-core/src/text/main/python_text.rs @@ -1,6 +1,8 @@ //! Select the Python string semantics of one CPython release. -use crate::text::_helpers::alnum_tables::{ALNUM_TABLES, ALPHA_TABLES, DECIMAL_TABLES}; +use crate::text::_helpers::alnum_tables::{ + ALNUM_TABLES, ALPHA_TABLES, DECIMAL_TABLES, PRINTABLE_TABLES, +}; use crate::text::_helpers::case_tables::{CASEFOLD_TABLES, IGNORECASE_KEY_TABLES, UPPER_TABLES}; use crate::text::models::{CleandocMargin, PythonText}; @@ -23,6 +25,10 @@ pub fn python_text(python_version: (u8, u8), unicode_version: &str) -> Option Option Option<(String, String)> { + let mut unknown: Vec<&str> = keys + .iter() + .filter(|(text, is_text)| !*is_text || !allowed.contains(text)) + .map(|(text, _)| text.as_str()) + .collect(); + if unknown.is_empty() { + return None; + } + unknown.sort_unstable(); + let mut allowed_sorted: Vec<&str> = allowed.iter().map(String::as_str).collect(); + allowed_sorted.sort_unstable(); + Some(( + format!( + "has unknown keys: {}; allowed keys: {}", + unknown.join(", "), + allowed_sorted.join(", ") + ), + unsupported_keys_help(&unknown, allowed), + )) +} diff --git a/crates/sqlbuild-discovery/src/key_checks/main/unsupported_keys_help.rs b/crates/sqlbuild-discovery/src/key_checks/main/unsupported_keys_help.rs new file mode 100644 index 0000000000..3ee677996d --- /dev/null +++ b/crates/sqlbuild-discovery/src/key_checks/main/unsupported_keys_help.rs @@ -0,0 +1,6 @@ +//! Python's `unsupported_keys_help`: nearest supported key suggestions. + +/// The help for `keys` that are not among `supported_keys`. +pub fn unsupported_keys_help(keys: &[&str], supported_keys: &[String]) -> String { + crate::_helpers::suggestions::unsupported_keys_help(keys, supported_keys) +} diff --git a/crates/sqlbuild-discovery/src/key_checks/mod.rs b/crates/sqlbuild-discovery/src/key_checks/mod.rs new file mode 100644 index 0000000000..e29b572755 --- /dev/null +++ b/crates/sqlbuild-discovery/src/key_checks/mod.rs @@ -0,0 +1,4 @@ +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/key_checks/tests.rs b/crates/sqlbuild-discovery/src/key_checks/tests.rs new file mode 100644 index 0000000000..8b89f249d8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/key_checks/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_unknown_mapping_keys.rs"] +mod unknown_mapping_keys; diff --git a/crates/sqlbuild-discovery/src/key_checks/tests/test_types.rs b/crates/sqlbuild-discovery/src/key_checks/tests/test_types.rs new file mode 100644 index 0000000000..231d62475d --- /dev/null +++ b/crates/sqlbuild-discovery/src/key_checks/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(super) struct UnknownKeysTestCase { + pub(super) description: &'static str, + pub(super) keys: Vec<(String, bool)>, + pub(super) expected_failure: Option<(&'static str, &'static str)>, +} diff --git a/crates/sqlbuild-discovery/src/key_checks/tests/test_unknown_mapping_keys.rs b/crates/sqlbuild-discovery/src/key_checks/tests/test_unknown_mapping_keys.rs new file mode 100644 index 0000000000..f53147765c --- /dev/null +++ b/crates/sqlbuild-discovery/src/key_checks/tests/test_unknown_mapping_keys.rs @@ -0,0 +1,43 @@ +use crate::key_checks::main::unknown_mapping_keys::unknown_mapping_keys; +use crate::key_checks::tests::test_types::UnknownKeysTestCase; + +#[test] +fn given_mapping_keys_when_checking_then_unknown_keys_are_sorted_with_a_suggestion() { + let test_cases = [ + UnknownKeysTestCase { + description: "allowed keys pass", + keys: vec![("name".to_owned(), true)], + expected_failure: None, + }, + UnknownKeysTestCase { + description: "a misspelt key is suggested", + keys: vec![("nme".to_owned(), true), ("name".to_owned(), true)], + expected_failure: Some(( + "has unknown keys: nme; allowed keys: columns, name", + "did you mean 'name'?", + )), + }, + UnknownKeysTestCase { + description: "a non-string key is never allowed", + keys: vec![("name".to_owned(), false)], + expected_failure: Some(( + "has unknown keys: name; allowed keys: columns, name", + "did you mean 'name'?", + )), + }, + ]; + let allowed = ["name".to_owned(), "columns".to_owned()]; + + for test_case in test_cases { + let failure = unknown_mapping_keys(&test_case.keys, &allowed); + + assert_eq!( + failure, + test_case + .expected_failure + .map(|(message, help)| (message.to_owned(), help.to_owned())), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index bca84d474c..c15f5b2b32 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -6,6 +6,7 @@ pub(crate) mod _helpers; pub mod constants; pub mod declaration_files; pub mod declarations; +pub mod key_checks; pub mod model_files; pub mod models; pub mod sql_tests; diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs b/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs index 51fa17cb8f..c441861ed8 100644 --- a/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs +++ b/crates/sqlbuild-discovery/src/tree/_helpers/listing.rs @@ -1,23 +1,40 @@ //! One directory listing with the entry type facts Python's `os.scandir` reports. use crate::tree::_helpers::raw_names::raw_segment; -use crate::tree::models::TreeEntry; +use crate::tree::models::{DirectoryListing, TreeEntry}; use std::fs::{DirEntry, FileType}; use std::path::Path; /// List a directory; an unreadable directory lists as empty, as in Python's snapshot. -pub(crate) fn list_directory(directory: &Path) -> Vec { +pub(crate) fn list_directory(directory: &Path) -> DirectoryListing { let Ok(entries) = std::fs::read_dir(directory) else { - return Vec::new(); + return DirectoryListing { + entries: Vec::new(), + interrupted: false, + }; }; + collect_listing(entries, |entry| tree_entry(&entry)) +} + +/// A listing that failed after the directory opened lists as empty and is marked interrupted. +pub(crate) fn collect_listing( + entries: impl Iterator>, + entry_facts: impl Fn(T) -> TreeEntry, +) -> DirectoryListing { let mut listing: Vec = Vec::new(); for entry in entries { let Ok(entry) = entry else { - return Vec::new(); + return DirectoryListing { + entries: Vec::new(), + interrupted: true, + }; }; - listing.push(tree_entry(&entry)); + listing.push(entry_facts(entry)); + } + DirectoryListing { + entries: listing, + interrupted: false, } - listing } fn tree_entry(entry: &DirEntry) -> TreeEntry { diff --git a/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs index 4ebf756e0a..2d1f18aa91 100644 --- a/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs +++ b/crates/sqlbuild-discovery/src/tree/_helpers/walking.rs @@ -5,7 +5,7 @@ use crate::tree::_helpers::listing::{join_relative, list_directory}; use crate::tree::models::{ProjectTree, TreeEntry}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use std::ffi::OsString; -use std::sync::Arc; +use std::sync::{Arc, PoisonError}; pub(crate) type WalkedDirectory = (String, Arc>); @@ -36,7 +36,14 @@ pub(crate) fn listing( if let Some(cached) = cached_listing(tree, directory) { return Ok(cached); } - let listing: Arc> = Arc::new(list_directory(&tree.absolute(directory))); + let read = list_directory(&tree.absolute(directory)); + if read.interrupted { + tree.interrupted + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(directory.to_owned()); + } + let listing: Arc> = Arc::new(read.entries); remember_raw_names(tree, directory, &listing); if let Ok(mut listings) = tree.listings.lock() { listings.insert(directory.to_owned(), Arc::clone(&listing)); diff --git a/crates/sqlbuild-discovery/src/tree/main/interrupted_listings.rs b/crates/sqlbuild-discovery/src/tree/main/interrupted_listings.rs new file mode 100644 index 0000000000..e6f8d2472e --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/main/interrupted_listings.rs @@ -0,0 +1,16 @@ +//! The directories whose listing failed after they opened, for callers that must report it. + +use std::sync::PoisonError; + +use crate::tree::models::ProjectTree; + +/// Relative directories read so far whose `readdir` failed mid-listing, sorted. +pub fn interrupted_listings(tree: &ProjectTree) -> Vec { + let mut directories: Vec = tree + .interrupted + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + directories.sort(); + directories +} diff --git a/crates/sqlbuild-discovery/src/tree/main/mod.rs b/crates/sqlbuild-discovery/src/tree/main/mod.rs index 643783dfed..f3c1887983 100644 --- a/crates/sqlbuild-discovery/src/tree/main/mod.rs +++ b/crates/sqlbuild-discovery/src/tree/main/mod.rs @@ -1,6 +1,7 @@ pub mod children; pub mod directories; pub mod display_text; +pub mod interrupted_listings; pub mod listings; pub mod path_order; pub mod rglob; diff --git a/crates/sqlbuild-discovery/src/tree/models.rs b/crates/sqlbuild-discovery/src/tree/models.rs index 90d333cbb3..6793ad48f1 100644 --- a/crates/sqlbuild-discovery/src/tree/models.rs +++ b/crates/sqlbuild-discovery/src/tree/models.rs @@ -22,6 +22,14 @@ pub struct TreeEntry { pub is_walkable_dir: bool, } +/// One directory's entries, and whether reading them failed after the directory opened. +#[derive(Debug)] +pub struct DirectoryListing { + pub entries: Vec, + /// `readdir` failed mid-listing; the listing is empty. Python 3.12's `Path.walk` raises here. + pub interrupted: bool, +} + /// Directory listings read once per pass and shared by every query, keyed by relative path. #[derive(Debug)] pub struct ProjectTree { @@ -29,6 +37,8 @@ pub struct ProjectTree { pub(crate) listings: Mutex>>>, /// The real names of listed entries that are not valid UTF-8, by relative path. pub(crate) raw_names: RwLock>, + /// Relative directories whose listing failed after they opened. + pub(crate) interrupted: Mutex>, } impl ProjectTree { @@ -38,6 +48,7 @@ impl ProjectTree { directory: directory.to_path_buf(), listings: Mutex::new(HashMap::new()), raw_names: RwLock::new(HashMap::new()), + interrupted: Mutex::new(Vec::new()), } } diff --git a/crates/sqlbuild-discovery/src/tree/tests.rs b/crates/sqlbuild-discovery/src/tree/tests.rs index 5775f2500c..00faf8ac4c 100644 --- a/crates/sqlbuild-discovery/src/tree/tests.rs +++ b/crates/sqlbuild-discovery/src/tree/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_interrupted_listings.rs"] +mod interrupted_listings; #[path = "tests/test_ordering.rs"] mod ordering; #[path = "tests/test_os_failures.rs"] diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_interrupted_listings.rs b/crates/sqlbuild-discovery/src/tree/tests/test_interrupted_listings.rs new file mode 100644 index 0000000000..f0e8adc047 --- /dev/null +++ b/crates/sqlbuild-discovery/src/tree/tests/test_interrupted_listings.rs @@ -0,0 +1,54 @@ +use std::io; + +use crate::tree::_helpers::listing::collect_listing; +use crate::tree::models::TreeEntry; +use crate::tree::tests::test_types::InterruptedListingTestCase; + +/// A `readdir` failure after the directory opened empties the listing and marks it interrupted. +#[test] +fn given_injected_listing_errors_when_collecting_then_listing_is_empty_and_interrupted() { + let test_cases = [ + InterruptedListingTestCase { + description: "a complete listing", + entries: &[Some("orders.sql"), Some("customers.sql")], + expected_names: &["orders.sql", "customers.sql"], + expected_interrupted: false, + }, + InterruptedListingTestCase { + description: "a failure after the first entry", + entries: &[Some("orders.sql"), None, Some("customers.sql")], + expected_names: &[], + expected_interrupted: true, + }, + InterruptedListingTestCase { + description: "a failure before any entry", + entries: &[None], + expected_names: &[], + expected_interrupted: true, + }, + ]; + for test_case in test_cases { + let results = test_case + .entries + .iter() + .map(|entry| entry.ok_or_else(|| io::Error::other("injected readdir failure"))); + let listing = collect_listing(results, |name| TreeEntry { + name: name.to_owned(), + raw_name: None, + segment: name.to_owned(), + is_dir: false, + is_walkable_dir: false, + }); + let names: Vec<&str> = listing + .entries + .iter() + .map(|entry| entry.name.as_str()) + .collect(); + assert_eq!(names, test_case.expected_names, "{}", test_case.description); + assert_eq!( + listing.interrupted, test_case.expected_interrupted, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/tree/tests/test_types.rs b/crates/sqlbuild-discovery/src/tree/tests/test_types.rs index bae31672cf..f69a3c5fef 100644 --- a/crates/sqlbuild-discovery/src/tree/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/tree/tests/test_types.rs @@ -73,3 +73,11 @@ pub(super) struct WideNameWalkTestCase { pub(super) expected_failures: &'static [bool], pub(super) expected_contents: &'static [&'static str], } + +/// Listing results fed to the listing collector, `None` standing for a failed `readdir` step. +pub(super) struct InterruptedListingTestCase { + pub(super) description: &'static str, + pub(super) entries: &'static [Option<&'static str>], + pub(super) expected_names: &'static [&'static str], + pub(super) expected_interrupted: bool, +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/_helpers/entry_names.rs b/crates/sqlbuild-discovery/src/yaml_files/_helpers/entry_names.rs new file mode 100644 index 0000000000..2e8fed8c8a --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/_helpers/entry_names.rs @@ -0,0 +1,77 @@ +//! The `\s*-\s*name:` entry prefix and every name `\s*(['"]?)\1\s*(#.*)?$` accepts. + +use std::collections::BTreeSet; + +use sqlbuild_core::text::main::is_python_space::is_python_space; + +const ENTRY_DASH: char = '-'; +const NAME_KEY: &[char] = &['n', 'a', 'm', 'e', ':']; +const COMMENT_START: char = '#'; +const QUOTES: [Option; 3] = [None, Some('\''), Some('"')]; + +fn leading_whitespace(characters: &[char], start: usize) -> usize { + let mut position: usize = start; + while position < characters.len() && is_python_space(characters[position]) { + position += 1; + } + position +} + +/// `re.match(r"(?P\s*)-\s*name:", line)`: the indent and where the value starts. +pub(crate) fn entry_prefix(line: &[char]) -> Option<(usize, usize)> { + let indent: usize = leading_whitespace(line, 0); + if line.get(indent) != Some(&ENTRY_DASH) { + return None; + } + let key_start: usize = leading_whitespace(line, indent + 1); + line[key_start..] + .starts_with(NAME_KEY) + .then_some((indent, key_start + NAME_KEY.len())) +} + +/// Positions after which only whitespace and an optional comment remain, in order. +fn closing_positions(value: &[char]) -> BTreeSet { + let mut positions: BTreeSet = BTreeSet::new(); + let mut tail: Option = Some(value.len()); + while let Some(end) = tail { + let mut position: usize = end; + positions.insert(position); + while position > 0 && is_python_space(value[position - 1]) { + position -= 1; + positions.insert(position); + } + tail = value[..end] + .iter() + .rposition(|character| *character == COMMENT_START); + } + positions +} + +/// Whether `value[at..]` starts with the quote, where no quote always matches. +fn quoted_at(value: &[char], at: usize, quote: Option) -> bool { + quote.is_none_or(|quote| value.get(at) == Some("e)) +} + +/// Every name the entry value accepts, in Python's generation order. +pub(crate) fn entry_names(value: &[char]) -> Vec { + let closings: BTreeSet = closing_positions(value); + let mut names: Vec = Vec::new(); + for start in 0..=leading_whitespace(value, 0) { + for quote in QUOTES { + if !quoted_at(value, start, quote) { + continue; + } + let width: usize = usize::from(quote.is_some()); + let body_start: usize = start + width; + for closing in &closings { + let Some(end) = closing.checked_sub(width) else { + continue; + }; + if end >= body_start && quoted_at(value, end, quote) { + names.push(value[body_start..end].iter().collect()); + } + } + } + } + names +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs index 6540a7ca43..e53d6ea07e 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod entry_names; pub(crate) mod failures; diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs index 5155a68f04..b6ed6a999a 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/main/mod.rs @@ -1,2 +1,3 @@ pub mod load_yaml_files; pub mod load_yaml_text; +pub mod yaml_entry_lines; diff --git a/crates/sqlbuild-discovery/src/yaml_files/main/yaml_entry_lines.rs b/crates/sqlbuild-discovery/src/yaml_files/main/yaml_entry_lines.rs new file mode 100644 index 0000000000..a51504bbe3 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/main/yaml_entry_lines.rs @@ -0,0 +1,32 @@ +//! The 1-based line of each outermost `- name:` entry of an authored YAML declaration file. + +use crate::yaml_files::_helpers::entry_names::{entry_names, entry_prefix}; +use sqlbuild_core::text::main::python_splitlines::python_splitlines; + +/// Every entry name with the line of its least-indented entry, earliest line on ties. +#[must_use] +pub fn yaml_entry_lines(contents: &str) -> Vec<(String, usize)> { + let mut best: Vec<(String, usize, usize)> = Vec::new(); + let mut positions: std::collections::HashMap = std::collections::HashMap::new(); + for (index, line) in python_splitlines(contents).into_iter().enumerate() { + let characters: Vec = line.chars().collect(); + let Some((indent, value_start)) = entry_prefix(&characters) else { + continue; + }; + for name in entry_names(&characters[value_start..]) { + match positions.get(&name) { + Some(position) if indent < best[*position].1 => { + best[*position] = (name, indent, index + 1); + } + Some(_) => {} + None => { + positions.insert(name.clone(), best.len()); + best.push((name, indent, index + 1)); + } + } + } + } + best.into_iter() + .map(|(name, _, line)| (name, line)) + .collect() +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests.rs b/crates/sqlbuild-discovery/src/yaml_files/tests.rs index 929b7b00fa..9026fd5196 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests.rs @@ -4,3 +4,5 @@ mod helpers; mod load_yaml_files; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_yaml_entry_lines.rs"] +mod yaml_entry_lines; diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs index f9cb105aab..1d06fe9305 100644 --- a/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_types.rs @@ -12,3 +12,10 @@ pub(super) struct YamlTextTestCase { /// The failure message and help, or `None` when the document loads. pub(super) expected_failure: Option<(&'static str, Option<&'static str>)>, } + +pub(super) struct EntryLinesTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + /// The line of each named entry the case checks; `None` where no entry has the name. + pub(super) expected_lines: &'static [(&'static str, Option)], +} diff --git a/crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_entry_lines.rs b/crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_entry_lines.rs new file mode 100644 index 0000000000..d6ca9e2953 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_files/tests/test_yaml_entry_lines.rs @@ -0,0 +1,58 @@ +use crate::yaml_files::main::yaml_entry_lines::yaml_entry_lines; +use crate::yaml_files::tests::test_types::EntryLinesTestCase; + +#[test] +fn given_yaml_declarations_when_indexing_entries_then_lines_match_the_python_scanner() { + let test_cases = [ + EntryLinesTestCase { + description: "the least-indented entry wins and keeps its value's leading space variant", + contents: " - name: x\n- name: y\n - name: x", + expected_lines: &[("x", Some(3)), (" x", Some(3)), ("y", Some(2)), ("z", None)], + }, + EntryLinesTestCase { + description: "equal indents keep the first line", + contents: "- name: orders\n- name: orders\n", + expected_lines: &[("orders", Some(1))], + }, + EntryLinesTestCase { + description: "quotes, comments and non-ASCII names on later lines", + contents: " - name: 'it''s' # note\n- name: \"caf\u{e9}\" # x", + expected_lines: &[ + ("it''s", Some(1)), + ("caf\u{e9}", Some(2)), + ("\"caf\u{e9}\" # x", Some(2)), + ], + }, + EntryLinesTestCase { + description: "CRLF and non-breaking-space indents count Unicode whitespace", + contents: "\u{a0}- name:orders\r\n-name: orders", + expected_lines: &[("orders", Some(2)), (" orders", Some(2))], + }, + EntryLinesTestCase { + description: "every hash starts a possible comment", + contents: "- name: a#b\n", + expected_lines: &[("a", Some(1)), ("a#b", Some(1))], + }, + EntryLinesTestCase { + description: "a key without a colon is not an entry and an empty value names nothing", + contents: "- name\n- name: \n", + expected_lines: &[("", Some(2)), (" ", Some(2)), ("name", None)], + }, + ]; + for test_case in test_cases { + let lines: Vec<(String, usize)> = yaml_entry_lines(test_case.contents); + + let found: Vec<(&str, Option)> = test_case + .expected_lines + .iter() + .map(|(name, _)| { + let line: Option = lines + .iter() + .find(|(entry, _)| entry == name) + .map(|(_, line)| *line); + (*name, line) + }) + .collect(); + assert_eq!(found, test_case.expected_lines, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-model-config/Cargo.toml b/crates/sqlbuild-model-config/Cargo.toml index 15f3596bb0..cf33c5a765 100644 --- a/crates/sqlbuild-model-config/Cargo.toml +++ b/crates/sqlbuild-model-config/Cargo.toml @@ -11,4 +11,5 @@ rust-version.workspace = true workspace = true [dependencies] +regex.workspace = true sqlbuild-core.workspace = true diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs index afb06b5ca1..1f894b4988 100644 --- a/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod macro_calls; +pub(crate) mod reuse; pub(crate) mod scan; pub(crate) mod template_tokens; diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/reuse.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/reuse.rs new file mode 100644 index 0000000000..69e56bb276 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/reuse.rs @@ -0,0 +1,31 @@ +//! Python's checks deciding whether a model's config can be reused across models. + +use crate::config_presence::_helpers::macro_calls::macro_call; +use crate::config_presence::_helpers::template_tokens::template_token; +use crate::types::{AuthoredNode, NodeKind}; + +/// `_contains_dynamic_or_unsupported_reusable_metadata`. +pub(crate) fn blocks_reuse(node: &N) -> bool { + match node.kind() { + NodeKind::Str => node + .with_text(|text| template_token(text) || macro_call(text)) + .unwrap_or(false), + NodeKind::Map => node + .entries() + .iter() + .any(|(key, value)| key.kind() != NodeKind::Str || blocks_reuse(value)), + NodeKind::List | NodeKind::Tuple => node.items().iter().any(blocks_reuse), + NodeKind::Null | NodeKind::Bool(_) | NodeKind::Int { .. } => false, + NodeKind::Other => node.number().is_none(), + } +} + +/// `_contains_mutable_nested_config`'s `is_mutable`. +pub(crate) fn mutable(node: &N) -> bool { + match node.kind() { + NodeKind::Map | NodeKind::List => true, + NodeKind::Tuple => node.items().iter().any(mutable), + NodeKind::Null | NodeKind::Bool(_) | NodeKind::Int { .. } | NodeKind::Str => false, + NodeKind::Other => node.number().is_none(), + } +} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/blocks_config_reuse.rs b/crates/sqlbuild-model-config/src/config_presence/main/blocks_config_reuse.rs new file mode 100644 index 0000000000..8bb7946978 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/main/blocks_config_reuse.rs @@ -0,0 +1,9 @@ +//! Whether model header metadata keeps a model from reusing another model's config. + +use crate::config_presence::_helpers::reuse::blocks_reuse; +use crate::types::AuthoredNode; + +/// Whether `value` holds a template, a macro call, a non-string key or an unsupported value. +pub fn blocks_config_reuse(value: &N) -> bool { + blocks_reuse(value) +} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs index 9cfc885af6..e99609df7a 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs @@ -1,2 +1,4 @@ +pub mod blocks_config_reuse; pub mod contains_template; pub mod first_macro_path; +pub mod mutable_config_value; diff --git a/crates/sqlbuild-model-config/src/config_presence/main/mutable_config_value.rs b/crates/sqlbuild-model-config/src/config_presence/main/mutable_config_value.rs new file mode 100644 index 0000000000..7eac36600c --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/main/mutable_config_value.rs @@ -0,0 +1,9 @@ +//! Whether an inherited config value is mutable, which keeps its config from being reused. + +use crate::config_presence::_helpers::reuse::mutable; +use crate::types::AuthoredNode; + +/// Whether `value` is a mapping, list, set or other object, or a tuple holding one. +pub fn mutable_config_value(value: &N) -> bool { + mutable(value) +} diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs index 40c033c0d8..c84da9a0c1 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs @@ -1,7 +1,9 @@ use crate::config_presence::_helpers::macro_calls::macro_call; use crate::config_presence::_helpers::scan::scan; +use crate::config_presence::main::blocks_config_reuse::blocks_config_reuse; use crate::config_presence::main::contains_template::contains_template; -use crate::config_presence::tests::test_types::PresenceTestCase; +use crate::config_presence::main::mutable_config_value::mutable_config_value; +use crate::config_presence::tests::test_types::{PresenceTestCase, ReuseTestCase}; use crate::tests::test_types::Value; #[test] @@ -80,3 +82,68 @@ fn given_config_values_when_scanning_then_presence_matches_python() { ); } } + +#[test] +fn given_config_values_when_checking_reuse_then_python_answers_are_returned() { + let test_cases = [ + ReuseTestCase { + description: "plain scalars neither block reuse nor are mutable", + value: Value::Tuple(vec![ + Value::Int(1), + Value::Bool(true), + Value::Null, + Value::Float, + ]), + expected_blocks_reuse: false, + expected_mutable: false, + }, + ReuseTestCase { + description: "a plain string is reusable", + value: Value::Str("Orders by customer."), + expected_blocks_reuse: false, + expected_mutable: false, + }, + ReuseTestCase { + description: "a template blocks reuse", + value: Value::Str("${var.label}"), + expected_blocks_reuse: true, + expected_mutable: false, + }, + ReuseTestCase { + description: "a macro call blocks reuse", + value: Value::Str("@label ()"), + expected_blocks_reuse: true, + expected_mutable: false, + }, + ReuseTestCase { + description: "a non-string mapping key blocks reuse and a mapping is mutable", + value: Value::Map(vec![(Value::Int(1), Value::Str("orders"))]), + expected_blocks_reuse: true, + expected_mutable: true, + }, + ReuseTestCase { + description: "an unsupported object blocks reuse and is mutable", + value: Value::Tuple(vec![Value::Date("2024-01-01")]), + expected_blocks_reuse: true, + expected_mutable: true, + }, + ReuseTestCase { + description: "a list of plain strings is reusable metadata but mutable", + value: Value::List(vec![Value::Str("orders")]), + expected_blocks_reuse: false, + expected_mutable: true, + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + blocks_config_reuse(&test_case.value), + mutable_config_value(&test_case.value) + ), + (test_case.expected_blocks_reuse, test_case.expected_mutable), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs index cb04f91f0a..9b46ae4c75 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs @@ -7,3 +7,11 @@ pub(super) struct PresenceTestCase { pub(super) expected_template: bool, pub(super) expected_macro_call: bool, } + +/// One config value and whether it blocks reuse as metadata and is mutable when inherited. +pub(super) struct ReuseTestCase { + pub(super) description: &'static str, + pub(super) value: Value, + pub(super) expected_blocks_reuse: bool, + pub(super) expected_mutable: bool, +} diff --git a/crates/sqlbuild-model-config/src/declarations/_helpers/enums.rs b/crates/sqlbuild-model-config/src/declarations/_helpers/enums.rs new file mode 100644 index 0000000000..8586f2e1be --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/_helpers/enums.rs @@ -0,0 +1,106 @@ +//! `_parse_enum_declaration` for model-header enums. + +use crate::declarations::_helpers::names::{declaration_name, identifier, is_identifier}; +use crate::declarations::constants::{INTEGER_TYPE, VARCHAR_TYPE}; +use crate::declarations::errors::DeclarationFailure; +use crate::declarations::models::{DeclarationSite, EnumFacts, EnumValue}; +use crate::sql_values::models::{RawKey, RawSqlValue}; + +type Members = Result, DeclarationFailure>; + +fn parse(message: String) -> DeclarationFailure { + DeclarationFailure::Parse(message) +} + +fn shorthand(raw: &[RawSqlValue], file_path: &str, enum_name: &str) -> Members { + let mut members: Vec<(String, EnumValue)> = Vec::new(); + for member in raw { + let RawSqlValue::Str(name) = member else { + return Err(parse(format!( + "{file_path} enum '{enum_name}' shorthand members must be identifiers" + ))); + }; + if !is_identifier(name) { + return Err(parse(format!( + "{file_path} enum '{enum_name}' shorthand members must be identifiers" + ))); + } + if members.iter().any(|(seen, _)| seen == name) { + return Err(parse(format!( + "{file_path} enum '{enum_name}' has duplicate member '{name}'" + ))); + } + members.push((name.clone(), EnumValue::Text(name.clone()))); + } + Ok(members) +} + +fn explicit(raw: &[(RawKey, RawSqlValue)], file_path: &str, enum_name: &str) -> Members { + raw.iter() + .map(|(key, value)| { + let key = match key { + RawKey::Str(text) => RawSqlValue::Str(text.clone()), + RawKey::Other(_) => RawSqlValue::Null, + }; + let name = identifier(&key, file_path, &format!("enum '{enum_name}' member"))?; + let value = match value { + RawSqlValue::Str(text) => EnumValue::Text(text.clone()), + RawSqlValue::Int(number) => EnumValue::Integer(number.to_string()), + RawSqlValue::BigInt(digits) => EnumValue::Integer(digits.clone()), + _ => { + return Err(parse(format!( + "{file_path} enum '{enum_name}' member '{name}' value must be a string or integer" + ))); + } + }; + Ok((name, value)) + }) + .collect() +} + +/// One validated model-header enum named by `raw_name`. +pub(crate) fn enum_declaration( + raw_name: &RawSqlValue, + raw_members: &RawSqlValue, + site: DeclarationSite<'_>, +) -> Result { + let file_path = site.file_path; + let name = declaration_name(raw_name, site, "enum")?; + let members = match raw_members { + RawSqlValue::List(items) => shorthand(items, file_path, &name)?, + RawSqlValue::Object(entries) => explicit(entries, file_path, &name)?, + _ => { + return Err(parse(format!( + "{file_path} enum '{name}' members must use [...] or (...)" + ))); + } + }; + let Some((_, first)) = members.first() else { + return Err(parse(format!( + "{file_path} enum '{name}' must declare at least one member" + ))); + }; + if let Some((member, _)) = members + .iter() + .find(|(member, _)| *member != member.to_ascii_uppercase()) + { + return Err(parse(format!( + "{file_path} enum '{name}' member identifiers must be uppercase: '{member}'" + ))); + } + let is_text = matches!(first, EnumValue::Text(_)); + if members + .iter() + .any(|(_, value)| matches!(value, EnumValue::Text(_)) != is_text) + { + return Err(parse(format!( + "{file_path} enum '{name}' members must use one consistent scalar type" + ))); + } + let scalar_type = if is_text { VARCHAR_TYPE } else { INTEGER_TYPE }; + Ok(EnumFacts { + name, + members, + scalar_type, + }) +} diff --git a/crates/sqlbuild-model-config/src/declarations/_helpers/mod.rs b/crates/sqlbuild-model-config/src/declarations/_helpers/mod.rs new file mode 100644 index 0000000000..536998a63e --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod enums; +pub(crate) mod names; +pub(crate) mod values; diff --git a/crates/sqlbuild-model-config/src/declarations/_helpers/names.rs b/crates/sqlbuild-model-config/src/declarations/_helpers/names.rs new file mode 100644 index 0000000000..a61d743323 --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/_helpers/names.rs @@ -0,0 +1,50 @@ +//! Declaration and member names: SQL identifiers and canonical resource identities. + +use crate::declarations::errors::DeclarationFailure; +use crate::declarations::models::DeclarationSite; +use crate::resource_identities::main::resource_identity_failure::resource_identity_failure; +use crate::sql_values::models::RawSqlValue; + +/// `^[A-Za-z_][A-Za-z0-9_]*$` under `fullmatch`. +pub(crate) fn is_identifier(text: &str) -> bool { + let mut characters = text.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +/// `_parse_identifier`: a SQL identifier named `label`. +pub(crate) fn identifier( + raw: &RawSqlValue, + file_path: &str, + label: &str, +) -> Result { + match raw { + RawSqlValue::Str(text) if is_identifier(text) => Ok(text.clone()), + _ => Err(DeclarationFailure::Parse(format!( + "{file_path} {label} name must be a SQL identifier" + ))), + } +} + +/// `_parse_declaration_name`: an identifier that is also a canonical resource identity. +pub(crate) fn declaration_name( + raw: &RawSqlValue, + site: DeclarationSite<'_>, + kind: &str, +) -> Result { + let name = identifier(raw, site.file_path, kind)?; + let private = site.model_name.is_some(); + if let Some((corrected, help)) = resource_identity_failure(&name, private) { + let owner = if private { "model-local" } else { "public" }; + return Err(DeclarationFailure::Identity( + format!( + "Invalid {owner} {kind} identity '{name}' in {}; use snake_case '{corrected}'", + site.file_path + ), + help, + )); + } + Ok(name) +} diff --git a/crates/sqlbuild-model-config/src/declarations/_helpers/values.rs b/crates/sqlbuild-model-config/src/declarations/_helpers/values.rs new file mode 100644 index 0000000000..c2df351e0f --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/_helpers/values.rs @@ -0,0 +1,133 @@ +//! `_parse_constant_declaration` and `typed_constant_declaration`. + +use crate::declarations::_helpers::names::declaration_name; +use crate::declarations::constants::{ + COLLECTION_RENDERINGS, RENDER_AS_OPTION, TYPE_OPTION, VALUE_OPTION, WRAPPER_OPTIONS, +}; +use crate::declarations::errors::DeclarationFailure; +use crate::declarations::models::{ConstantFacts, DeclarationSite}; +use crate::sql_values::errors::SqlValueFailure; +use crate::sql_values::main::normalize_sql_value::normalize_sql_value; +use crate::sql_values::models::{RawSqlValue, SqlValueLimits}; +use sqlbuild_core::models::{SqlValue, SqlValueKind}; + +fn parse(message: String) -> DeclarationFailure { + DeclarationFailure::Parse(message) +} + +/// `_parse_constant_option`: an absent option, or a non-empty string. +fn option( + raw: Option<&RawSqlValue>, + file_path: &str, + label: &str, +) -> Result, DeclarationFailure> { + match raw { + None => Ok(None), + Some(RawSqlValue::Str(text)) if !text.is_empty() => Ok(Some(text.clone())), + Some(_) => Err(parse(format!("{file_path} {label} must be an identifier"))), + } +} + +/// `typed_constant_declaration`: the typed value of a constant whose name and options are valid. +pub(crate) fn typed_constant( + name: &str, + raw: &RawSqlValue, + options: (Option<&str>, Option<&str>), + file_path: &str, +) -> Result { + let (explicit_type, render_as) = options; + let render_as = match render_as { + None => None, + Some(text) => Some( + COLLECTION_RENDERINGS + .iter() + .copied() + .find(|known| *known == text) + .ok_or_else(|| { + parse(format!( + "{file_path} constant '{name}' render_as must be value_list or array" + )) + })?, + ), + }; + let context = format!("{file_path} constant '{name}'"); + let value: SqlValue = + normalize_sql_value(raw, &context, explicit_type, SqlValueLimits::default()).map_err( + |failure| match failure { + SqlValueFailure::Invalid(message) => parse(message), + SqlValueFailure::DecimalOverflow => DeclarationFailure::DecimalOverflow, + }, + )?; + let kind = value.logical_type().kind; + if let Some(rendering) = render_as + && !matches!(kind, SqlValueKind::List | SqlValueKind::Set) + { + return Err(parse(format!( + "{file_path} constant '{name}' is {} and does not support render_as {rendering}", + kind.as_str() + ))); + } + Ok(ConstantFacts { + name: name.to_owned(), + value, + render_as, + }) +} + +fn wrapper_option<'a>( + arguments: &'a [(String, RawSqlValue)], + key: &str, +) -> Option<&'a RawSqlValue> { + arguments + .iter() + .rev() + .find(|(name, _)| name == key) + .map(|(_, value)| value) +} + +/// A model-header constant: a plain value or a `constant(value := ..., type := ..., ...)` call. +pub(crate) fn header_constant( + raw_name: &RawSqlValue, + raw_value: &RawSqlValue, + site: DeclarationSite<'_>, +) -> Result { + let file_path = site.file_path; + let name = declaration_name(raw_name, site, "constant")?; + let RawSqlValue::Call(arguments) = raw_value else { + return typed_constant(&name, raw_value, (None, None), file_path); + }; + let mut unknown: Vec<&str> = arguments + .iter() + .map(|(key, _)| key.as_str()) + .filter(|key| !WRAPPER_OPTIONS.contains(key)) + .collect(); + unknown.sort_unstable(); + unknown.dedup(); + if !unknown.is_empty() { + return Err(parse(format!( + "{file_path} constant '{name}' wrapper has unknown keys: {}", + unknown.join(", ") + ))); + } + let Some(value) = wrapper_option(arguments, VALUE_OPTION) else { + return Err(parse(format!( + "{file_path} constant '{name}' is missing required value" + ))); + }; + let explicit = option( + wrapper_option(arguments, TYPE_OPTION), + file_path, + &format!("constant '{name}' type"), + )?; + let render_as = option( + wrapper_option(arguments, RENDER_AS_OPTION), + file_path, + &format!("constant '{name}' render_as"), + )?; + typed_constant( + &name, + value, + (explicit.as_deref(), render_as.as_deref()), + file_path, + ) +} diff --git a/crates/sqlbuild-model-config/src/declarations/constants.rs b/crates/sqlbuild-model-config/src/declarations/constants.rs new file mode 100644 index 0000000000..cf96334c8b --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/constants.rs @@ -0,0 +1,10 @@ +//! Declaration option names, scalar SQL types and rendering choices. + +pub(crate) const VALUE_OPTION: &str = "value"; +pub(crate) const TYPE_OPTION: &str = "type"; +pub(crate) const RENDER_AS_OPTION: &str = "render_as"; +pub(crate) const WRAPPER_OPTIONS: [&str; 3] = ["value", "type", "render_as"]; +pub(crate) const VARCHAR_TYPE: &str = "VARCHAR"; +pub(crate) const INTEGER_TYPE: &str = "INTEGER"; +/// `CollectionRendering` values. +pub(crate) const COLLECTION_RENDERINGS: [&str; 2] = ["value_list", "array"]; diff --git a/crates/sqlbuild-model-config/src/declarations/errors.rs b/crates/sqlbuild-model-config/src/declarations/errors.rs new file mode 100644 index 0000000000..28a94c4c11 --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/errors.rs @@ -0,0 +1,12 @@ +//! Why a declaration is rejected. + +/// The exception a declaration failure raises. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DeclarationFailure { + /// `DeclarationParseError` with this message. + Parse(String), + /// `ResourceIdentityError` with its message and help. + Identity(String, String), + /// `decimal.Overflow` from normalizing a set of decimals. + DecimalOverflow, +} diff --git a/crates/sqlbuild-model-config/src/declarations/main/mod.rs b/crates/sqlbuild-model-config/src/declarations/main/mod.rs new file mode 100644 index 0000000000..4c549240a3 --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/main/mod.rs @@ -0,0 +1,3 @@ +pub mod model_constant_declarations; +pub mod model_enum_declarations; +pub mod typed_constant_declaration; diff --git a/crates/sqlbuild-model-config/src/declarations/main/model_constant_declarations.rs b/crates/sqlbuild-model-config/src/declarations/main/model_constant_declarations.rs new file mode 100644 index 0000000000..d517b01cf8 --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/main/model_constant_declarations.rs @@ -0,0 +1,33 @@ +//! Python's `parse_model_constant_declarations`. + +use crate::declarations::_helpers::values::header_constant; +use crate::declarations::errors::DeclarationFailure; +use crate::declarations::models::{ConstantFacts, DeclarationSite}; +use crate::sql_values::models::{RawKey, RawSqlValue}; + +/// The validated constants of a model header's `constants` value. +pub fn model_constant_declarations( + raw: Option<&RawSqlValue>, + site: DeclarationSite<'_>, +) -> Result, DeclarationFailure> { + let entries = match raw { + None | Some(RawSqlValue::Null) => return Ok(Vec::new()), + Some(RawSqlValue::Object(entries)) => entries, + Some(_) => { + return Err(DeclarationFailure::Parse(format!( + "{} model 'constants' must be a mapping", + site.file_path + ))); + } + }; + entries + .iter() + .map(|(key, value)| { + let name = match key { + RawKey::Str(text) => RawSqlValue::Str(text.clone()), + RawKey::Other(_) => RawSqlValue::Null, + }; + header_constant(&name, value, site) + }) + .collect() +} diff --git a/crates/sqlbuild-model-config/src/declarations/main/model_enum_declarations.rs b/crates/sqlbuild-model-config/src/declarations/main/model_enum_declarations.rs new file mode 100644 index 0000000000..09a242c0ff --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/main/model_enum_declarations.rs @@ -0,0 +1,33 @@ +//! Python's `parse_model_enum_declarations`. + +use crate::declarations::_helpers::enums::enum_declaration; +use crate::declarations::errors::DeclarationFailure; +use crate::declarations::models::{DeclarationSite, EnumFacts}; +use crate::sql_values::models::{RawKey, RawSqlValue}; + +/// The validated enums of a model header's `enums` value. +pub fn model_enum_declarations( + raw: Option<&RawSqlValue>, + site: DeclarationSite<'_>, +) -> Result, DeclarationFailure> { + let entries = match raw { + None | Some(RawSqlValue::Null) => return Ok(Vec::new()), + Some(RawSqlValue::Object(entries)) => entries, + Some(_) => { + return Err(DeclarationFailure::Parse(format!( + "{} model 'enums' must be a mapping", + site.file_path + ))); + } + }; + entries + .iter() + .map(|(key, members)| { + let name = match key { + RawKey::Str(text) => RawSqlValue::Str(text.clone()), + RawKey::Other(_) => RawSqlValue::Null, + }; + enum_declaration(&name, members, site) + }) + .collect() +} diff --git a/crates/sqlbuild-model-config/src/declarations/main/typed_constant_declaration.rs b/crates/sqlbuild-model-config/src/declarations/main/typed_constant_declaration.rs new file mode 100644 index 0000000000..af605ce34a --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/main/typed_constant_declaration.rs @@ -0,0 +1,16 @@ +//! Python's `typed_constant_declaration` for a constant file entry. + +use crate::declarations::_helpers::values::typed_constant; +use crate::declarations::errors::DeclarationFailure; +use crate::declarations::models::ConstantFacts; +use crate::sql_values::models::RawSqlValue; + +/// The typed value of constant `name` with its optional explicit type and rendering. +pub fn typed_constant_declaration( + name: &str, + raw: &RawSqlValue, + options: (Option<&str>, Option<&str>), + file_path: &str, +) -> Result { + typed_constant(name, raw, options, file_path) +} diff --git a/crates/sqlbuild-model-config/src/declarations/mod.rs b/crates/sqlbuild-model-config/src/declarations/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/declarations/models.rs b/crates/sqlbuild-model-config/src/declarations/models.rs new file mode 100644 index 0000000000..0d486cfb41 --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/models.rs @@ -0,0 +1,33 @@ +//! Validated enum and constant declarations. + +use sqlbuild_core::models::SqlValue; + +/// An enum member value: text, or an integer in its decimal digits. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EnumValue { + Text(String), + Integer(String), +} + +/// `EnumDeclaration` without its path and owner. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EnumFacts { + pub name: String, + pub members: Vec<(String, EnumValue)>, + pub scalar_type: &'static str, +} + +/// `ConstantDeclaration` without its path and owner. +#[derive(Clone, Debug, PartialEq)] +pub struct ConstantFacts { + pub name: String, + pub value: SqlValue, + pub render_as: Option<&'static str>, +} + +/// Where declarations are authored: the path diagnostics name and the owning model, if any. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DeclarationSite<'a> { + pub file_path: &'a str, + pub model_name: Option<&'a str>, +} diff --git a/crates/sqlbuild-model-config/src/declarations/tests.rs b/crates/sqlbuild-model-config/src/declarations/tests.rs new file mode 100644 index 0000000000..22767e48ff --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_declarations.rs"] +mod declarations; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/declarations/tests/helpers.rs b/crates/sqlbuild-model-config/src/declarations/tests/helpers.rs new file mode 100644 index 0000000000..af8f91175c --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/tests/helpers.rs @@ -0,0 +1,6 @@ +use crate::sql_values::models::{RawKey, RawSqlValue}; + +/// A header `enums` mapping with one enum. +pub(super) fn enums(name: &str, members: RawSqlValue) -> RawSqlValue { + RawSqlValue::Object(vec![(RawKey::Str(name.to_owned()), members)]) +} diff --git a/crates/sqlbuild-model-config/src/declarations/tests/test_declarations.rs b/crates/sqlbuild-model-config/src/declarations/tests/test_declarations.rs new file mode 100644 index 0000000000..812284a7f7 --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/tests/test_declarations.rs @@ -0,0 +1,75 @@ +use crate::declarations::errors::DeclarationFailure; +use crate::declarations::main::model_enum_declarations::model_enum_declarations; +use crate::declarations::models::DeclarationSite; +use crate::declarations::tests::helpers::enums; +use crate::declarations::tests::test_types::HeaderEnumTestCase; +use crate::sql_values::models::{RawKey, RawSqlValue}; + +#[test] +fn given_model_header_enums_when_validating_then_python_rules_apply() { + let test_cases = [ + HeaderEnumTestCase { + description: "shorthand members are text", + raw: enums( + "_status", + RawSqlValue::List(vec![RawSqlValue::Str("ACTIVE".to_owned())]), + ), + expected_scalar_types: Ok(vec!["VARCHAR"]), + }, + HeaderEnumTestCase { + description: "explicit integer members", + raw: enums( + "_status", + RawSqlValue::Object(vec![( + RawKey::Str("ACTIVE".to_owned()), + RawSqlValue::Int(1), + )]), + ), + expected_scalar_types: Ok(vec!["INTEGER"]), + }, + HeaderEnumTestCase { + description: "model-local enums must be private snake_case identities", + raw: enums( + "Status", + RawSqlValue::List(vec![RawSqlValue::Str("ACTIVE".to_owned())]), + ), + expected_scalar_types: Err(DeclarationFailure::Identity( + "Invalid model-local enum identity 'Status' in models/a.sql; use snake_case \ + '_status'" + .to_owned(), + crate::header_metadata::constants::IDENTITY_HELP.to_owned(), + )), + }, + HeaderEnumTestCase { + description: "member identifiers are uppercase", + raw: enums( + "_status", + RawSqlValue::List(vec![RawSqlValue::Str("active".to_owned())]), + ), + expected_scalar_types: Err(DeclarationFailure::Parse( + "models/a.sql enum '_status' member identifiers must be uppercase: 'active'" + .to_owned(), + )), + }, + ]; + + for test_case in test_cases { + let site = DeclarationSite { + file_path: "models/a.sql", + model_name: Some("a"), + }; + + let declared = model_enum_declarations(Some(&test_case.raw), site).map(|facts| { + facts + .iter() + .map(|fact| fact.scalar_type) + .collect::>() + }); + + assert_eq!( + declared, test_case.expected_scalar_types, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/declarations/tests/test_types.rs b/crates/sqlbuild-model-config/src/declarations/tests/test_types.rs new file mode 100644 index 0000000000..2ded2b2746 --- /dev/null +++ b/crates/sqlbuild-model-config/src/declarations/tests/test_types.rs @@ -0,0 +1,8 @@ +use crate::declarations::errors::DeclarationFailure; +use crate::sql_values::models::RawSqlValue; + +pub(super) struct HeaderEnumTestCase { + pub(super) description: &'static str, + pub(super) raw: RawSqlValue, + pub(super) expected_scalar_types: Result, DeclarationFailure>, +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/dynamic.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/dynamic.rs new file mode 100644 index 0000000000..7e6e2b73e1 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/dynamic.rs @@ -0,0 +1,127 @@ +//! `dynamic_columns` families read as `parse_dynamic_column_families` reads them. + +use crate::errors::ConfigError; +use crate::header_metadata::_helpers::text::non_blank_text; +use crate::header_metadata::constants::{ + DYNAMIC_FAMILY_KEYS, DYNAMIC_NAME_PATTERN_KEY, DYNAMIC_REQUIRED_KEYS, +}; +use crate::header_metadata::models::DynamicFamily; +use crate::name_patterns::main::compile_name_pattern::compile_name_pattern; +use crate::types::{AuthoredNode, NodeKind}; + +/// Who declares the families. +pub(crate) struct DynamicSource<'a> { + pub(crate) model_name: &'a str, + pub(crate) path: &'a str, +} + +pub(crate) fn dynamic_families( + raw: &N, + source: &DynamicSource<'_>, +) -> Result, ConfigError> { + let model_name: &str = source.model_name; + if raw.kind() == NodeKind::Null { + return Ok(Vec::new()); + } + let entries: Vec<(N, N)> = if raw.kind() == NodeKind::Map { + raw.entries() + } else { + Vec::new() + }; + if entries.is_empty() { + return Err(ConfigError::compile(format!( + "model '{model_name}' dynamic_columns must be a non-empty named block in {}", + source.path + ))); + } + let mut families: Vec = Vec::with_capacity(entries.len()); + for (name, family) in &entries { + families.push(dynamic_family(name, family, source)?); + } + if families.len() > 1 && families.iter().any(|family| family.name_pattern.is_none()) { + return Err(ConfigError::compile(format!( + "model '{model_name}' declares multiple dynamic column families; each family requires \ + name_pattern so runtime columns can be classified unambiguously" + ))); + } + Ok(families) +} + +fn dynamic_family( + name: &N, + family: &N, + source: &DynamicSource<'_>, +) -> Result { + let model_name: &str = source.model_name; + let Some(name) = name.text().filter(|text| !text.is_empty()) else { + return Err(ConfigError::compile(format!( + "model '{model_name}' dynamic column family names must be non-empty strings" + ))); + }; + if family.kind() != NodeKind::Map { + return Err(ConfigError::compile(format!( + "model '{model_name}' dynamic column family '{name}' must be a block" + ))); + } + let mut values: Vec<(String, N)> = Vec::new(); + for (key, value) in family.entries() { + let key: String = key.python_str(); + values.retain(|(existing, _)| *existing != key); + values.push((key, value)); + } + let value = |wanted: &str| { + values + .iter() + .find(|(key, _)| key == wanted) + .map(|(_, value)| value.clone()) + }; + let mut unknown: Vec<&str> = values + .iter() + .map(|(key, _)| key.as_str()) + .filter(|key| !DYNAMIC_FAMILY_KEYS.contains(key)) + .collect(); + if !unknown.is_empty() { + unknown.sort_unstable(); + return Err(ConfigError::compile(format!( + "model '{model_name}' dynamic column family '{name}' has unsupported keys: {}", + unknown.join(", ") + ))); + } + let required = |key: &str| { + value(key).as_ref().and_then(non_blank_text).ok_or_else(|| { + ConfigError::compile(format!( + "model '{model_name}' dynamic column family '{name}' requires a non-empty {key}" + )) + }) + }; + let [pivot_key, value_key, aggregate_key, type_key] = DYNAMIC_REQUIRED_KEYS; + let pivot_column: String = required(pivot_key)?; + let value_column: String = required(value_key)?; + let aggregate: String = required(aggregate_key)?; + let data_type: String = required(type_key)?; + let name_pattern: Option = match value(DYNAMIC_NAME_PATTERN_KEY) { + None => None, + Some(pattern) if pattern.kind() == NodeKind::Null => None, + Some(pattern) => Some(non_blank_text(&pattern).ok_or_else(|| { + ConfigError::compile(format!( + "model '{model_name}' dynamic column family '{name}' name_pattern must be a \ + non-empty regular expression" + )) + })?), + }; + if let Some(pattern) = &name_pattern + && let Err(error) = compile_name_pattern(pattern) + { + return Err(ConfigError::compile(format!( + "model '{model_name}' dynamic column family '{name}' has invalid name_pattern: {error}" + ))); + } + Ok(DynamicFamily { + name, + pivot_column, + value_column, + aggregate: aggregate.to_uppercase(), + data_type, + name_pattern, + }) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/factories.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/factories.rs new file mode 100644 index 0000000000..f06713d391 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/factories.rs @@ -0,0 +1,86 @@ +//! `audit_factories` references read as `parse_model_header_audit_factories` reads them. + +use crate::errors::ConfigError; +use crate::header_metadata::_helpers::text::non_blank_text; +use crate::types::{AuthoredNode, NodeKind}; + +pub(crate) fn factory_references( + raw: &N, + model_name: &str, + path: &str, + available: &[String], +) -> Result, ConfigError> { + if raw.kind() == NodeKind::Null { + return Ok(Vec::new()); + } + if raw.kind() != NodeKind::List { + return Err(ConfigError::compile(format!( + "{path} model audit_factories must be a list" + ))); + } + let mut references: Vec = Vec::new(); + for entry in raw.items() { + let Some(value) = non_blank_text(&entry) else { + return Err(ConfigError::compile(format!( + "{path} model audit_factories entries must be bare identifiers" + ))); + }; + if !is_bare_identifier(&value) { + return Err(ConfigError::compile(format!( + "{path} model audit_factories entry '{value}' must be a bare identifier" + ))); + } + if references.contains(&value) { + return Err(ConfigError::compile(format!( + "model '{model_name}' in {path} references audit factory '{value}' more than once" + ))); + } + references.push(value); + } + if let Some(unknown) = references + .iter() + .find(|reference| !available.contains(reference)) + { + let mut names: Vec<&str> = available.iter().map(String::as_str).collect(); + names.sort_unstable(); + let listed: String = if names.is_empty() { + "none discovered".to_owned() + } else { + names.join(", ") + }; + return Err(ConfigError::compile(format!( + "model '{model_name}' in {path} references unknown audit factory '{unknown}'. \ + Available audit factories: {listed}" + ))); + } + Ok(references) +} + +/// The first generated audit whose identity a direct or earlier generated audit already uses. +pub(crate) fn duplicate_identity( + direct: &[String], + generated: &[String], + model_name: &str, + path: &str, +) -> Result<(), ConfigError> { + let mut seen: Vec<&str> = direct.iter().map(String::as_str).collect(); + for identity in generated { + if seen.contains(&identity.as_str()) { + return Err(ConfigError::compile(format!( + "model '{model_name}' in {path} has duplicate audit case name '{identity}' across \ + direct and factory attachments" + ))); + } + seen.push(identity); + } + Ok(()) +} + +/// Python's `re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value)`. +fn is_bare_identifier(value: &str) -> bool { + let mut characters = value.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs index b32e40efc8..dc71ab4c6b 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs @@ -1,79 +1,16 @@ //! `validate_resource_identity`, with its suggested snake_case spelling. use crate::errors::{ConfigError, ErrorClass}; -use sqlbuild_core::text::main::python_strip::python_strip; - -use crate::header_metadata::constants::{FALLBACK_IDENTITY, IDENTITY_HELP}; +use crate::resource_identities::main::resource_identity_failure::resource_identity_failure; /// Raise `ResourceIdentityError` unless `name` is canonical snake_case. pub(crate) fn check_identity(name: &str, kind: &str, path: &str) -> Result<(), ConfigError> { - if is_snake_case(name) { + let Some((suggestion, help)) = resource_identity_failure(name, false) else { return Ok(()); - } - let suggestion = suggested_identity(name); + }; Err(ConfigError::compile(format!( "Invalid {kind} identity '{name}' in {path}; use snake_case '{suggestion}'" )) .with_class(ErrorClass::ResourceIdentity) - .with_help(IDENTITY_HELP)) -} - -/// Return whether `name` fully matches `^[a-z](?:[a-z0-9_]*[a-z0-9])?$`. -fn is_snake_case(name: &str) -> bool { - let bytes = name.as_bytes(); - match (bytes.first(), bytes.last()) { - (Some(first), Some(last)) => { - first.is_ascii_lowercase() - && (last.is_ascii_lowercase() || last.is_ascii_digit()) - && bytes - .iter() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_') - } - _ => false, - } -} - -/// Return `suggest_resource_identity` for a public identity; its patterns are ASCII-only. -fn suggested_identity(name: &str) -> String { - let characters: Vec = python_strip(name).chars().collect(); - let acronyms = split_where(&characters, |index| { - characters[index - 1].is_ascii_uppercase() - && characters[index].is_ascii_uppercase() - && characters - .get(index + 1) - .is_some_and(char::is_ascii_lowercase) - }); - let words = split_where(&acronyms, |index| { - (acronyms[index - 1].is_ascii_lowercase() || acronyms[index - 1].is_ascii_digit()) - && acronyms[index].is_ascii_uppercase() - }); - let mut replaced = String::with_capacity(words.len()); - let mut in_invalid_run = false; - for character in words { - if character.is_ascii_alphanumeric() || character == '_' { - replaced.push(character.to_ascii_lowercase()); - in_invalid_run = false; - } else if !in_invalid_run { - replaced.push('_'); - in_invalid_run = true; - } - } - let corrected = replaced.trim_matches('_'); - if corrected.is_empty() { - FALLBACK_IDENTITY.to_owned() - } else { - corrected.to_owned() - } -} - -/// Insert `_` before every index after the first where `boundary` holds. -fn split_where(characters: &[char], boundary: impl Fn(usize) -> bool) -> Vec { - let mut result: Vec = Vec::with_capacity(characters.len() * 2); - for (index, character) in characters.iter().enumerate() { - if index > 0 && boundary(index) { - result.push('_'); - } - result.push(*character); - } - result + .with_help(help)) } diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs index 2f9c85d8b4..e65d8e5c83 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs @@ -1,5 +1,7 @@ pub(crate) mod audits; pub(crate) mod columns; +pub(crate) mod dynamic; +pub(crate) mod factories; pub(crate) mod identity; pub(crate) mod text; pub(crate) mod thresholds; diff --git a/crates/sqlbuild-model-config/src/header_metadata/constants.rs b/crates/sqlbuild-model-config/src/header_metadata/constants.rs index 56f8b23239..9294c25752 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/constants.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/constants.rs @@ -33,3 +33,15 @@ pub const FALLBACK_IDENTITY: &str = "resource_name"; pub const MODEL_LABEL: &str = "model"; /// The code of the nullable column that also declares `not_null`. pub const NULLABLE_NOT_NULL_CODE: &str = "P002"; +/// Keys one `dynamic_columns` family may declare. +pub const DYNAMIC_FAMILY_KEYS: [&str; 5] = [ + "pivot_column", + "value_column", + "aggregate", + "type", + "name_pattern", +]; +/// The family keys that must hold non-blank text, in the order they are checked. +pub const DYNAMIC_REQUIRED_KEYS: [&str; 4] = ["pivot_column", "value_column", "aggregate", "type"]; +/// The optional regular expression that classifies a family's runtime columns. +pub const DYNAMIC_NAME_PATTERN_KEY: &str = "name_pattern"; diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/audit_factory_references.rs b/crates/sqlbuild-model-config/src/header_metadata/main/audit_factory_references.rs new file mode 100644 index 0000000000..17f6418c0b --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/audit_factory_references.rs @@ -0,0 +1,14 @@ +//! Validate one MODEL header's `audit_factories` against the discovered factories. + +use crate::errors::ConfigError; +use crate::header_metadata::_helpers::factories::factory_references; +use crate::types::AuthoredNode; + +/// The referenced factory names in order; `model` is `(model name, file path)`. +pub fn audit_factory_references( + raw: &N, + model: (&str, &str), + available: &[String], +) -> Result, ConfigError> { + factory_references(raw, model.0, model.1, available) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/merged_audit_identities.rs b/crates/sqlbuild-model-config/src/header_metadata/main/merged_audit_identities.rs new file mode 100644 index 0000000000..6d7292069d --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/merged_audit_identities.rs @@ -0,0 +1,13 @@ +//! Reject factory audits whose identity a model's other audits already use. + +use crate::errors::ConfigError; +use crate::header_metadata::_helpers::factories::duplicate_identity; + +/// `Ok` when every generated identity is new; `model` is `(model name, file path)`. +pub fn merged_audit_identities( + direct: &[String], + generated: &[String], + model: (&str, &str), +) -> Result<(), ConfigError> { + duplicate_identity(direct, generated, model.0, model.1) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs b/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs index 03b4f0af13..91f6ab14fa 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/main/mod.rs @@ -1 +1,5 @@ +pub mod audit_factory_references; +pub mod merged_audit_identities; +pub mod parse_dynamic_columns; pub mod parse_header_metadata; +pub mod parse_measurement_policy; diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/parse_dynamic_columns.rs b/crates/sqlbuild-model-config/src/header_metadata/main/parse_dynamic_columns.rs new file mode 100644 index 0000000000..8107fe96dd --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/parse_dynamic_columns.rs @@ -0,0 +1,15 @@ +//! Parse one MODEL header's `dynamic_columns`, with the errors Python raises. + +use crate::errors::ConfigError; +use crate::header_metadata::_helpers::dynamic::{DynamicSource, dynamic_families}; +use crate::header_metadata::models::DynamicFamily; +use crate::types::AuthoredNode; + +/// The families `raw` declares, each `name_pattern` compiled by the native matcher. +pub fn parse_dynamic_columns( + raw: &N, + model: (&str, &str), +) -> Result, ConfigError> { + let (model_name, path) = model; + dynamic_families(raw, &DynamicSource { model_name, path }) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/main/parse_measurement_policy.rs b/crates/sqlbuild-model-config/src/header_metadata/main/parse_measurement_policy.rs new file mode 100644 index 0000000000..4ed7ab9141 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/main/parse_measurement_policy.rs @@ -0,0 +1,28 @@ +//! A standalone measurement audit header's `thresholds` and `minimum_samples`. + +use crate::errors::ConfigError; +use crate::header_metadata::_helpers::text::Site; +use crate::header_metadata::_helpers::thresholds::thresholds; +use crate::header_metadata::models::ParsedThresholds; +use crate::types::{AuthoredNode, NodeKind}; + +/// Parsed thresholds and the authored `minimum_samples` node, or the error at `path`. +pub fn parse_measurement_policy( + policy: (Option<&N>, Option<&N>), + path: &str, + label: &str, +) -> Result<(Option, Option), ConfigError> { + let site = Site { + path, + label, + column: None, + }; + let (raw_thresholds, raw_minimum_samples) = policy; + let parsed: Option = thresholds(raw_thresholds, site)?; + let minimum_samples: Option = match raw_minimum_samples.map(AuthoredNode::kind) { + None | Some(NodeKind::Null) => None, + Some(NodeKind::Int { negative: false }) => raw_minimum_samples.cloned(), + Some(_) => return Err(site.error("'minimum_samples' must be a non-negative integer")), + }; + Ok((parsed, minimum_samples)) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/models.rs b/crates/sqlbuild-model-config/src/header_metadata/models.rs index 3a6e5b63bb..28a9a38619 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/models.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/models.rs @@ -52,3 +52,14 @@ pub struct HeaderMetadata { pub columns: Result>, ConfigError>, pub audits: Result>, ConfigError>, } + +/// One `dynamic_columns` family; `aggregate` is upper-cased as Python stores it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DynamicFamily { + pub name: String, + pub pivot_column: String, + pub value_column: String, + pub aggregate: String, + pub data_type: String, + pub name_pattern: Option, +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests.rs b/crates/sqlbuild-model-config/src/header_metadata/tests.rs index 4c3e7b55db..48e279791e 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/tests.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_header_extensions.rs"] +mod header_extensions; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_parse_header_metadata.rs"] diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/helpers.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/helpers.rs index 46629791b0..7f65fbea0f 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/tests/helpers.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/helpers.rs @@ -84,3 +84,47 @@ fn text(value: Option<&Value>) -> String { .and_then(AuthoredNode::text) .unwrap_or_else(|| "-".to_owned()) } + +/// Families as `name:aggregate:pattern`. +pub(super) fn dynamic_summary(raw: &Value) -> Result, String> { + crate::header_metadata::main::parse_dynamic_columns::parse_dynamic_columns( + raw, + ("orders", "models/orders.sql"), + ) + .map(|families| { + families + .iter() + .map(|family| { + format!( + "{}:{}:{}", + family.name, + family.aggregate, + family.name_pattern.as_deref().unwrap_or("-") + ) + }) + .collect() + }) + .map_err(stop_summary) +} + +/// References of `orders` against the discovered `daily_checks` and `freshness` factories. +pub(super) fn factory_summary(raw: &Value) -> Result, String> { + crate::header_metadata::main::audit_factory_references::audit_factory_references( + raw, + ("orders", "models/orders.sql"), + &["freshness".to_owned(), "daily_checks".to_owned()], + ) + .map_err(stop_summary) +} + +/// One revenue family, with `pattern` as its `name_pattern` when given. +pub(super) fn family(pattern: Option<&'static str>) -> Value { + let entries = [ + ("pivot_column", Value::Str("channel")), + ("value_column", Value::Str("amount")), + ("aggregate", Value::Str("sum")), + ("type", Value::Str("DECIMAL")), + ]; + let pattern = pattern.map(|pattern| ("name_pattern", Value::Str(pattern))); + map(&entries.into_iter().chain(pattern).collect::>()) +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/test_header_extensions.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/test_header_extensions.rs new file mode 100644 index 0000000000..406b9798f5 --- /dev/null +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/test_header_extensions.rs @@ -0,0 +1,118 @@ +use crate::header_metadata::tests::helpers::{dynamic_summary, factory_summary, family, map}; +use crate::header_metadata::tests::test_types::{ + DynamicColumnsTestCase, FactoryReferencesTestCase, +}; +use crate::tests::test_types::Value; + +#[test] +fn given_dynamic_columns_when_parsing_then_families_or_first_error_are_returned() { + let test_cases = [ + DynamicColumnsTestCase { + description: "one family without a pattern", + raw: map(&[("revenue", family(None))]), + expected_families: Ok(&["revenue:SUM:-"]), + }, + DynamicColumnsTestCase { + description: "empty block", + raw: map(&[]), + expected_families: Err( + "model 'orders' dynamic_columns must be a non-empty named block in models/orders.sql", + ), + }, + DynamicColumnsTestCase { + description: "unsupported keys are sorted", + raw: map(&[( + "revenue", + map(&[("zone", Value::Null), ("alias", Value::Null)]), + )]), + expected_families: Err( + "model 'orders' dynamic column family 'revenue' has unsupported keys: alias, zone", + ), + }, + DynamicColumnsTestCase { + description: "blank required key", + raw: map(&[("revenue", map(&[("pivot_column", Value::Str(" "))]))]), + expected_families: Err( + "model 'orders' dynamic column family 'revenue' requires a non-empty pivot_column", + ), + }, + DynamicColumnsTestCase { + description: "invalid pattern", + raw: map(&[("revenue", family(Some("(")))]), + expected_families: Err( + "model 'orders' dynamic column family 'revenue' has invalid name_pattern: \ + unclosed group", + ), + }, + DynamicColumnsTestCase { + description: "two families need patterns", + raw: map(&[("revenue", family(Some("rev_.*"))), ("units", family(None))]), + expected_families: Err( + "model 'orders' declares multiple dynamic column families; each family requires \ + name_pattern so runtime columns can be classified unambiguously", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + dynamic_summary(&test_case.raw), + test_case + .expected_families + .map(|families| families.iter().map(|family| (*family).to_owned()).collect()) + .map_err(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_audit_factories_when_resolving_then_references_or_first_error_are_returned() { + let test_cases = [ + FactoryReferencesTestCase { + description: "known factories in order", + raw: Value::List(vec![Value::Str("freshness"), Value::Str("daily_checks")]), + expected_references: Ok(&["freshness", "daily_checks"]), + }, + FactoryReferencesTestCase { + description: "a tuple is not a list", + raw: Value::Tuple(vec![Value::Str("freshness")]), + expected_references: Err("models/orders.sql model audit_factories must be a list"), + }, + FactoryReferencesTestCase { + description: "dotted entry", + raw: Value::List(vec![Value::Str("pkg.freshness")]), + expected_references: Err( + "models/orders.sql model audit_factories entry 'pkg.freshness' must be a bare identifier", + ), + }, + FactoryReferencesTestCase { + description: "repeated entry", + raw: Value::List(vec![Value::Str("freshness"), Value::Str("freshness")]), + expected_references: Err( + "model 'orders' in models/orders.sql references audit factory 'freshness' more than once", + ), + }, + FactoryReferencesTestCase { + description: "unknown entry lists sorted factories", + raw: Value::List(vec![Value::Str("volume")]), + expected_references: Err( + "model 'orders' in models/orders.sql references unknown audit factory 'volume'. \ + Available audit factories: daily_checks, freshness", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + factory_summary(&test_case.raw), + test_case + .expected_references + .map(|references| references.iter().map(|name| (*name).to_owned()).collect()) + .map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs b/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs index 8e0f831b52..201454f1e0 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/tests/test_types.rs @@ -8,3 +8,18 @@ pub(super) struct HeaderMetadataTestCase { /// Lines per column then model audit, or the first error message, or `unsupported`. pub(super) expected_summary: Result<&'static [&'static str], &'static str>, } + +/// One header `dynamic_columns` value and its families or first error. +pub(super) struct DynamicColumnsTestCase { + pub(super) description: &'static str, + pub(super) raw: Value, + /// `name:aggregate:pattern` per family, or the first error message. + pub(super) expected_families: Result<&'static [&'static str], &'static str>, +} + +/// One header `audit_factories` value and its references or first error. +pub(super) struct FactoryReferencesTestCase { + pub(super) description: &'static str, + pub(super) raw: Value, + pub(super) expected_references: Result<&'static [&'static str], &'static str>, +} diff --git a/crates/sqlbuild-model-config/src/lib.rs b/crates/sqlbuild-model-config/src/lib.rs index 5e8b132f8e..a34311abe3 100644 --- a/crates/sqlbuild-model-config/src/lib.rs +++ b/crates/sqlbuild-model-config/src/lib.rs @@ -3,10 +3,15 @@ #![forbid(unsafe_code)] pub mod config_presence; +pub mod declarations; pub mod errors; pub mod header_metadata; pub mod model_validation; +pub mod name_patterns; pub mod path_defaults; +pub mod resource_identities; +pub mod sql_tests; +pub mod sql_values; pub mod templates; pub mod types; diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index b15366a917..fb42a7fa9e 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -1,11 +1,12 @@ //! Contract, materialization, storage, migration and placeholder rules with Python's errors. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use sqlbuild_core::text::main::is_python_word::is_python_word; use sqlbuild_core::text::main::python_strip::python_strip; use sqlbuild_core::text::models::PythonText; +use crate::errors::ConfigError; use crate::model_validation::_helpers::config::python_lower; use crate::model_validation::_helpers::config::{ ConfigView, one_of, sorted_values, text_if_string, @@ -16,6 +17,10 @@ use crate::model_validation::constants::{ MICROBATCH_CONCURRENCY_HELP, MICROBATCH_CONCURRENCY_NOTE, OLD_NAME_VIEW_KEY, PLACEHOLDER_SIGIL, TABLE_BACKED_MATERIALIZATIONS, VIEW_MATERIALIZATION, }; +use crate::model_validation::constants::{ + CHAINED_HELP, CLAIMED_HELP, MATERIALIZATION_HELP, MIGRATABLE_MATERIALIZATIONS, ORIGIN_HELP, + QUALIFIED_NAME_SEPARATOR, +}; use crate::model_validation::models::{ ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; @@ -286,3 +291,79 @@ fn placeholder_names(python: PythonText, sql: &str) -> BTreeSet { } names } + +/// Column renames need a history model, or a table or view that migrates itself. +pub(crate) fn column_materialization_problem( + model_name: &str, + materialized: Option<&str>, + model_migrates: bool, + first: &str, +) -> Result<(), ConfigError> { + let history: bool = materialized.is_some_and(|value| HISTORY_MATERIALIZATIONS.contains(&value)); + let migratable: bool = + materialized.is_some_and(|value| MIGRATABLE_MATERIALIZATIONS.contains(&value)); + if !history && (!migratable || !model_migrates) { + let shown: &str = materialized.unwrap_or("None"); + return Err(ConfigError::compile(format!( + "model '{model_name}': column '{first}' declares migrate_from, which is only valid \ + for incremental and snapshot models, or for table and view models that declare \ + migrate_from themselves (the old name's compatibility view then presents the column \ + under its old name); '{shown}' models are rebuilt with their new columns" + )) + .with_help(MATERIALIZATION_HELP)); + } + Ok(()) +} + +/// Each origin names one other column, claimed by only one rename. +pub(crate) fn origin_problem( + model_name: &str, + declared: &[(&str, &str)], +) -> Result<(), ConfigError> { + let mut sources: HashMap = HashMap::new(); + for &(name, origin) in declared { + let origin: &str = python_strip(origin); + if origin.is_empty() || origin.contains(QUALIFIED_NAME_SEPARATOR) { + return Err(ConfigError::compile(format!( + "model '{model_name}': column '{name}' migrate_from must name one column of the \ + model's existing table" + )) + .with_help(ORIGIN_HELP)); + } + if python_lower(origin) == python_lower(name) { + return Err(ConfigError::compile(format!( + "model '{model_name}': column '{name}' migrate_from cannot name the column itself" + ))); + } + if let Some(claimed_by) = sources.get(&python_lower(origin)) { + return Err(ConfigError::compile(format!( + "model '{model_name}': columns '{claimed_by}' and '{name}' both declare \ + migrate_from {origin}; a column can be renamed to only one new name" + )) + .with_help(CLAIMED_HELP)); + } + sources.insert(python_lower(origin), name); + } + Ok(()) +} + +/// No rename may start from another renamed column. +pub(crate) fn chained_problem( + model_name: &str, + declared: &[(&str, &str)], +) -> Result<(), ConfigError> { + let destinations: HashMap = declared + .iter() + .map(|&(name, _)| (python_lower(name), name)) + .collect(); + for &(name, origin) in declared { + if destinations.contains_key(&python_lower(python_strip(origin))) { + return Err(ConfigError::compile(format!( + "model '{model_name}': column '{name}' migrates from '{origin}', which itself \ + declares migrate_from; chained or swapped column renames are not supported" + )) + .with_help(CHAINED_HELP)); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index 6da4c88b15..0249e5c8c0 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -265,3 +265,17 @@ pub(crate) const PYTHON_314: u8 = 14; pub(crate) const MAX_DURATION_AMOUNT: u128 = i64::MAX as u128; /// The MODEL header key of the retention override. pub(crate) const RETENTION_KEY: &str = "time_travel_retention"; + +/// Materializations whose columns can be renamed when the model also migrates. +pub(crate) const MIGRATABLE_MATERIALIZATIONS: [&str; 4] = + ["table", "view", "incremental", "snapshot"]; +pub(crate) const QUALIFIED_NAME_SEPARATOR: char = '.'; +pub(crate) const MATERIALIZATION_HELP: &str = + "remove migrate_from from the column, or declare the model's own migrate_from"; +pub(crate) const ORIGIN_HELP: &str = + "write the old column name, for example revenue (migrate_from amount)"; +pub(crate) const CLAIMED_HELP: &str = + "keep migrate_from on the column that should receive the existing data"; +pub(crate) const CHAINED_HELP: &str = "a chain or swap cannot be renamed in place safely; give the new column \ + a name no other renamed column uses, or rebuild the model with \ + --full-refresh"; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/mod.rs b/crates/sqlbuild-model-config/src/model_validation/main/mod.rs index 106d25d285..8c47253af8 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/mod.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/mod.rs @@ -1,4 +1,5 @@ pub mod retention_override; pub mod table_type_override; +pub mod validate_column_migrations; pub mod validate_model_config; pub mod validate_model_references; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/validate_column_migrations.rs b/crates/sqlbuild-model-config/src/model_validation/main/validate_column_migrations.rs new file mode 100644 index 0000000000..6fd1b913d4 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/main/validate_column_migrations.rs @@ -0,0 +1,28 @@ +//! Validate a model's column renames before they are attached to its schema entry. + +use crate::errors::ConfigError; +use crate::model_validation::_helpers::materialization::{ + chained_problem, column_materialization_problem, origin_problem, +}; +use crate::model_validation::models::ColumnMigration; + +/// The first problem with the columns' `migrate_from` declarations, if any. +pub fn validate_column_migrations( + model_name: &str, + materialized: Option<&str>, + model_migrates: bool, + columns: &[ColumnMigration], +) -> Result<(), ConfigError> { + let mut declared: Vec<(&str, &str)> = Vec::new(); + for column in columns { + if let Some(origin) = column.migrate_from.as_deref() { + declared.push((column.name.as_str(), origin)); + } + } + let Some(&(first, _)) = declared.first() else { + return Ok(()); + }; + column_materialization_problem(model_name, materialized, model_migrates, first)?; + origin_problem(model_name, &declared)?; + chained_problem(model_name, &declared) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/models.rs b/crates/sqlbuild-model-config/src/model_validation/models.rs index c6b198dd21..0c009bcdbe 100644 --- a/crates/sqlbuild-model-config/src/model_validation/models.rs +++ b/crates/sqlbuild-model-config/src/model_validation/models.rs @@ -92,3 +92,10 @@ pub enum TableTypeOverride { Permanent, Transient, } + +/// One header column's name and the old column it renames, if any. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ColumnMigration { + pub name: String, + pub migrate_from: Option, +} diff --git a/crates/sqlbuild-model-config/src/model_validation/tests.rs b/crates/sqlbuild-model-config/src/model_validation/tests.rs index 5fbe948817..ebddf13611 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_column_migrations.rs"] +mod column_migrations; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_storage_overrides.rs"] diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs index 91ac128294..a300270610 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs @@ -115,3 +115,28 @@ pub(super) fn snapshot(extra: Vec<(&'static str, Value)>) -> Vec<(&'static str, config.extend(extra); config } + +/// The first column-rename error message for `orders` with these columns, if any. +pub(super) fn column_migration_message( + materialized: Option<&str>, + model_migrates: bool, + columns: &[(&str, Option<&str>)], +) -> Option { + let columns: Vec = columns + .iter() + .map( + |(name, origin)| crate::model_validation::models::ColumnMigration { + name: (*name).to_owned(), + migrate_from: origin.map(str::to_owned), + }, + ) + .collect(); + crate::model_validation::main::validate_column_migrations::validate_column_migrations( + "orders", + materialized, + model_migrates, + &columns, + ) + .err() + .map(|error| error.message) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_column_migrations.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_column_migrations.rs new file mode 100644 index 0000000000..dff5ee2923 --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_column_migrations.rs @@ -0,0 +1,71 @@ +use crate::model_validation::tests::helpers::column_migration_message; +use crate::model_validation::tests::test_types::ColumnMigrationTestCase; + +#[test] +fn given_column_renames_when_validating_then_first_problem_is_reported() { + let test_cases = [ + ColumnMigrationTestCase { + description: "incremental rename is accepted", + materialized: Some("incremental"), + model_migrates: false, + columns: &[("revenue", Some("amount"))], + expected_message: None, + }, + ColumnMigrationTestCase { + description: "table without its own migrate_from is rebuilt", + materialized: Some("table"), + model_migrates: false, + columns: &[("revenue", Some("amount"))], + expected_message: Some( + "model 'orders': column 'revenue' declares migrate_from, which is only valid for \ + incremental and snapshot models, or for table and view models that declare \ + migrate_from themselves (the old name's compatibility view then presents the \ + column under its old name); 'table' models are rebuilt with their new columns", + ), + }, + ColumnMigrationTestCase { + description: "qualified origin", + materialized: Some("incremental"), + model_migrates: false, + columns: &[("revenue", Some("orders.amount"))], + expected_message: Some( + "model 'orders': column 'revenue' migrate_from must name one column of the \ + model's existing table", + ), + }, + ColumnMigrationTestCase { + description: "two columns claim one origin", + materialized: Some("incremental"), + model_migrates: false, + columns: &[("revenue", Some("amount")), ("total", Some("AMOUNT"))], + expected_message: Some( + "model 'orders': columns 'revenue' and 'total' both declare migrate_from AMOUNT; \ + a column can be renamed to only one new name", + ), + }, + ColumnMigrationTestCase { + description: "swapped renames", + materialized: Some("snapshot"), + model_migrates: false, + columns: &[("left_id", Some("right_id")), ("right_id", Some("left_id"))], + expected_message: Some( + "model 'orders': column 'left_id' migrates from 'right_id', which itself declares \ + migrate_from; chained or swapped column renames are not supported", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + column_migration_message( + test_case.materialized, + test_case.model_migrates, + test_case.columns + ) + .as_deref(), + test_case.expected_message, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_types.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_types.rs index 669a7675df..a9768ad56b 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_types.rs @@ -25,3 +25,13 @@ pub(super) struct TableTypeOverrideTestCase { pub(super) value: Option, pub(super) expected_override: Result, } + +/// One model's column renames and the first error validation reports, if any. +pub(super) struct ColumnMigrationTestCase { + pub(super) description: &'static str, + pub(super) materialized: Option<&'static str>, + pub(super) model_migrates: bool, + /// `(column, migrate_from)` per column. + pub(super) columns: &'static [(&'static str, Option<&'static str>)], + pub(super) expected_message: Option<&'static str>, +} diff --git a/crates/sqlbuild-model-config/src/name_patterns/constants.rs b/crates/sqlbuild-model-config/src/name_patterns/constants.rs new file mode 100644 index 0000000000..ab26fe58fa --- /dev/null +++ b/crates/sqlbuild-model-config/src/name_patterns/constants.rs @@ -0,0 +1,8 @@ +//! How a dynamic column family's `name_pattern` is anchored and described. + +/// Anchors a pattern so it must match a whole column name, like Python's `re.fullmatch`. +pub(crate) const FULL_MATCH_PREFIX: &str = r"\A(?:"; +pub(crate) const FULL_MATCH_SUFFIX: &str = r")\z"; +/// The line of a regex syntax error that names the problem. +pub(crate) const SYNTAX_ERROR_PREFIX: &str = "error: "; +pub(crate) const TOO_LARGE_MESSAGE: &str = "the pattern is too large to compile"; diff --git a/crates/sqlbuild-model-config/src/name_patterns/main/compile_name_pattern.rs b/crates/sqlbuild-model-config/src/name_patterns/main/compile_name_pattern.rs new file mode 100644 index 0000000000..9aabda0bc0 --- /dev/null +++ b/crates/sqlbuild-model-config/src/name_patterns/main/compile_name_pattern.rs @@ -0,0 +1,22 @@ +//! Compile a dynamic column family's `name_pattern` to match whole column names. + +use crate::name_patterns::constants::{ + FULL_MATCH_PREFIX, FULL_MATCH_SUFFIX, SYNTAX_ERROR_PREFIX, TOO_LARGE_MESSAGE, +}; +use crate::name_patterns::models::NamePattern; +use regex::Regex; + +/// The whole-name matcher for `pattern`, or a one-line reason it is invalid or unsupported. +pub fn compile_name_pattern(pattern: &str) -> Result { + let compiled = Regex::new(&format!("{FULL_MATCH_PREFIX}{pattern}{FULL_MATCH_SUFFIX}")); + compiled + .map(|regex| NamePattern { regex }) + .map_err(|error| match error { + regex::Error::Syntax(detail) => detail + .lines() + .find_map(|line| line.strip_prefix(SYNTAX_ERROR_PREFIX)) + .unwrap_or(detail.as_str()) + .to_owned(), + _ => TOO_LARGE_MESSAGE.to_owned(), + }) +} diff --git a/crates/sqlbuild-model-config/src/name_patterns/main/mod.rs b/crates/sqlbuild-model-config/src/name_patterns/main/mod.rs new file mode 100644 index 0000000000..0f5573a483 --- /dev/null +++ b/crates/sqlbuild-model-config/src/name_patterns/main/mod.rs @@ -0,0 +1 @@ +pub mod compile_name_pattern; diff --git a/crates/sqlbuild-model-config/src/name_patterns/mod.rs b/crates/sqlbuild-model-config/src/name_patterns/mod.rs new file mode 100644 index 0000000000..17d3abdfe6 --- /dev/null +++ b/crates/sqlbuild-model-config/src/name_patterns/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/name_patterns/models.rs b/crates/sqlbuild-model-config/src/name_patterns/models.rs new file mode 100644 index 0000000000..c8b8b34a2b --- /dev/null +++ b/crates/sqlbuild-model-config/src/name_patterns/models.rs @@ -0,0 +1,16 @@ +//! A compiled dynamic column family `name_pattern`. + +use regex::Regex; + +/// A `name_pattern` compiled to match whole column names. +#[derive(Clone, Debug)] +pub struct NamePattern { + pub(crate) regex: Regex, +} + +impl NamePattern { + /// Whether `column_name` wholly matches the pattern. + pub fn matches(&self, column_name: &str) -> bool { + self.regex.is_match(column_name) + } +} diff --git a/crates/sqlbuild-model-config/src/name_patterns/tests.rs b/crates/sqlbuild-model-config/src/name_patterns/tests.rs new file mode 100644 index 0000000000..a2e0a18d2c --- /dev/null +++ b/crates/sqlbuild-model-config/src/name_patterns/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_compile_name_pattern.rs"] +mod compile_name_pattern; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/name_patterns/tests/test_compile_name_pattern.rs b/crates/sqlbuild-model-config/src/name_patterns/tests/test_compile_name_pattern.rs new file mode 100644 index 0000000000..b0bd871fcb --- /dev/null +++ b/crates/sqlbuild-model-config/src/name_patterns/tests/test_compile_name_pattern.rs @@ -0,0 +1,58 @@ +use crate::name_patterns::main::compile_name_pattern::compile_name_pattern; +use crate::name_patterns::tests::test_types::NamePatternTestCase; + +#[test] +fn given_name_patterns_when_matching_columns_then_whole_names_must_match() { + let test_cases = [ + NamePatternTestCase { + description: "an anchored pattern accepts a whole lowercase name", + pattern: "^[a-z]+$", + column_name: "books", + expected_match: Ok(true), + }, + NamePatternTestCase { + description: "an unanchored pattern must still match the whole name", + pattern: "amount_[0-9]+", + column_name: "total_amount_12", + expected_match: Ok(false), + }, + NamePatternTestCase { + description: "alternation is grouped before anchoring", + pattern: "books|games", + column_name: "games", + expected_match: Ok(true), + }, + NamePatternTestCase { + description: "a trailing newline is not part of a whole-name match", + pattern: "[a-z]+", + column_name: "books\n", + expected_match: Ok(false), + }, + NamePatternTestCase { + description: "an unclosed group is rejected", + pattern: "(", + column_name: "books", + expected_match: Err("unclosed group"), + }, + NamePatternTestCase { + description: "look-around is rejected as unsupported", + pattern: "(?=books)books", + column_name: "books", + expected_match: Err( + "look-around, including look-ahead and look-behind, is not supported", + ), + }, + ]; + + for test_case in test_cases { + let matched = compile_name_pattern(test_case.pattern) + .map(|pattern| pattern.matches(test_case.column_name)); + + assert_eq!( + matched, + test_case.expected_match.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/name_patterns/tests/test_types.rs b/crates/sqlbuild-model-config/src/name_patterns/tests/test_types.rs new file mode 100644 index 0000000000..ca54f76beb --- /dev/null +++ b/crates/sqlbuild-model-config/src/name_patterns/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(super) struct NamePatternTestCase { + pub(super) description: &'static str, + pub(super) pattern: &'static str, + pub(super) column_name: &'static str, + pub(super) expected_match: Result, +} diff --git a/crates/sqlbuild-model-config/src/resource_identities/_helpers/mod.rs b/crates/sqlbuild-model-config/src/resource_identities/_helpers/mod.rs new file mode 100644 index 0000000000..e0963126d2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/resource_identities/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod spelling; diff --git a/crates/sqlbuild-model-config/src/resource_identities/_helpers/spelling.rs b/crates/sqlbuild-model-config/src/resource_identities/_helpers/spelling.rs new file mode 100644 index 0000000000..1c1f8f64b5 --- /dev/null +++ b/crates/sqlbuild-model-config/src/resource_identities/_helpers/spelling.rs @@ -0,0 +1,63 @@ +//! `validate_resource_identity`'s canonical snake_case check and suggested spelling. + +use sqlbuild_core::text::main::python_strip::python_strip; + +/// Return whether `name` fully matches `^[a-z](?:[a-z0-9_]*[a-z0-9])?$`. +pub(crate) fn is_snake_case(name: &str) -> bool { + let bytes = name.as_bytes(); + match (bytes.first(), bytes.last()) { + (Some(first), Some(last)) => { + first.is_ascii_lowercase() + && (last.is_ascii_lowercase() || last.is_ascii_digit()) + && bytes + .iter() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_') + } + _ => false, + } +} + +/// Return `suggest_resource_identity` for a public identity; its patterns are ASCII-only. +pub(crate) fn suggested_identity(name: &str) -> String { + let characters: Vec = python_strip(name).chars().collect(); + let acronyms = split_where(&characters, |index| { + characters[index - 1].is_ascii_uppercase() + && characters[index].is_ascii_uppercase() + && characters + .get(index + 1) + .is_some_and(char::is_ascii_lowercase) + }); + let words = split_where(&acronyms, |index| { + (acronyms[index - 1].is_ascii_lowercase() || acronyms[index - 1].is_ascii_digit()) + && acronyms[index].is_ascii_uppercase() + }); + let mut replaced = String::with_capacity(words.len()); + let mut in_invalid_run = false; + for character in words { + if character.is_ascii_alphanumeric() || character == '_' { + replaced.push(character.to_ascii_lowercase()); + in_invalid_run = false; + } else if !in_invalid_run { + replaced.push('_'); + in_invalid_run = true; + } + } + let corrected = replaced.trim_matches('_'); + if corrected.is_empty() { + crate::header_metadata::constants::FALLBACK_IDENTITY.to_owned() + } else { + corrected.to_owned() + } +} + +/// Insert `_` before every index after the first where `boundary` holds. +fn split_where(characters: &[char], boundary: impl Fn(usize) -> bool) -> Vec { + let mut result: Vec = Vec::with_capacity(characters.len() * 2); + for (index, character) in characters.iter().enumerate() { + if index > 0 && boundary(index) { + result.push('_'); + } + result.push(*character); + } + result +} diff --git a/crates/sqlbuild-model-config/src/resource_identities/main/mod.rs b/crates/sqlbuild-model-config/src/resource_identities/main/mod.rs new file mode 100644 index 0000000000..ceb06e9f2c --- /dev/null +++ b/crates/sqlbuild-model-config/src/resource_identities/main/mod.rs @@ -0,0 +1 @@ +pub mod resource_identity_failure; diff --git a/crates/sqlbuild-model-config/src/resource_identities/main/resource_identity_failure.rs b/crates/sqlbuild-model-config/src/resource_identities/main/resource_identity_failure.rs new file mode 100644 index 0000000000..f2b4aefb32 --- /dev/null +++ b/crates/sqlbuild-model-config/src/resource_identities/main/resource_identity_failure.rs @@ -0,0 +1,23 @@ +//! Python's `validate_resource_identity`: canonical lowercase snake_case resource identities. + +use crate::header_metadata::constants::IDENTITY_HELP; +use crate::resource_identities::_helpers::spelling::{is_snake_case, suggested_identity}; + +/// The snake_case spelling to suggest for `name` and the rename help, or `None` when canonical. +pub fn resource_identity_failure(name: &str, private_identity: bool) -> Option<(String, String)> { + let canonical = if private_identity { + name.strip_prefix('_').is_some_and(is_snake_case) + } else { + is_snake_case(name) + }; + if canonical { + return None; + } + let suggestion = suggested_identity(name); + let corrected = if private_identity { + format!("_{suggestion}") + } else { + suggestion + }; + Some((corrected, IDENTITY_HELP.to_owned())) +} diff --git a/crates/sqlbuild-model-config/src/resource_identities/mod.rs b/crates/sqlbuild-model-config/src/resource_identities/mod.rs new file mode 100644 index 0000000000..668a17d0aa --- /dev/null +++ b/crates/sqlbuild-model-config/src/resource_identities/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod _helpers; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/resource_identities/tests.rs b/crates/sqlbuild-model-config/src/resource_identities/tests.rs new file mode 100644 index 0000000000..5ea4bc33bd --- /dev/null +++ b/crates/sqlbuild-model-config/src/resource_identities/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_resource_identity_failure.rs"] +mod resource_identity_failure; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/resource_identities/tests/test_resource_identity_failure.rs b/crates/sqlbuild-model-config/src/resource_identities/tests/test_resource_identity_failure.rs new file mode 100644 index 0000000000..c91e00b876 --- /dev/null +++ b/crates/sqlbuild-model-config/src/resource_identities/tests/test_resource_identity_failure.rs @@ -0,0 +1,55 @@ +use crate::resource_identities::main::resource_identity_failure::resource_identity_failure; +use crate::resource_identities::tests::test_types::ResourceIdentityTestCase; + +#[test] +fn given_identities_when_validating_then_only_snake_case_passes_with_python_suggestions() { + let test_cases = [ + ResourceIdentityTestCase { + description: "canonical public identity", + name: "orders__daily2", + private_identity: false, + expected_suggestion: None, + }, + ResourceIdentityTestCase { + description: "acronym and word boundaries", + name: " HTTPStatusCode ", + private_identity: false, + expected_suggestion: Some("http_status_code"), + }, + ResourceIdentityTestCase { + description: "invalid runs collapse and edges trim", + name: "-Orders--éx_", + private_identity: false, + expected_suggestion: Some("orders_x"), + }, + ResourceIdentityTestCase { + description: "private identities keep their underscore", + name: "_customer_ids", + private_identity: true, + expected_suggestion: None, + }, + ResourceIdentityTestCase { + description: "a private identity is suggested with its underscore", + name: "CustomerIds", + private_identity: true, + expected_suggestion: Some("_customer_ids"), + }, + ResourceIdentityTestCase { + description: "nothing valid falls back", + name: "***", + private_identity: false, + expected_suggestion: Some("resource_name"), + }, + ]; + + for test_case in test_cases { + let failure = resource_identity_failure(test_case.name, test_case.private_identity); + + assert_eq!( + failure.map(|(suggestion, _)| suggestion), + test_case.expected_suggestion.map(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/resource_identities/tests/test_types.rs b/crates/sqlbuild-model-config/src/resource_identities/tests/test_types.rs new file mode 100644 index 0000000000..c796ddb9ea --- /dev/null +++ b/crates/sqlbuild-model-config/src/resource_identities/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(super) struct ResourceIdentityTestCase { + pub(super) description: &'static str, + pub(super) name: &'static str, + pub(super) private_identity: bool, + pub(super) expected_suggestion: Option<&'static str>, +} diff --git a/crates/sqlbuild-model-config/src/sql_tests/_helpers/header.rs b/crates/sqlbuild-model-config/src/sql_tests/_helpers/header.rs new file mode 100644 index 0000000000..6a83509a44 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/_helpers/header.rs @@ -0,0 +1,81 @@ +//! TEST header fields: name, mode, analysis flag and cursor bounds. + +use crate::sql_tests::_helpers::mode::defined_modes; +use crate::sql_tests::_helpers::parameters::entry; +use crate::sql_tests::constants::{ + MODE_KEY, MODEL_MODE, NAME_KEY, SQL_ANALYSIS_KEY, SQL_TEST_MODES, +}; +use crate::sql_values::models::{RawKey, RawSqlValue}; +use sqlbuild_core::text::main::python_strip::python_strip; + +pub(crate) fn cursor_bound( + header: &[(RawKey, RawSqlValue)], + key: &str, + file_path: &str, +) -> Result, String> { + match entry(header, key) { + None | Some(RawSqlValue::Null) => Ok(None), + Some(RawSqlValue::Int(number)) => Ok(Some(number.to_string())), + Some(RawSqlValue::BigInt(digits)) => Ok(Some(digits.clone())), + Some(RawSqlValue::Str(text)) if !python_strip(text).is_empty() => { + Ok(Some(python_strip(text).to_owned())) + } + Some(_) => Err(format!( + "TEST() {key} in '{file_path}' must be a non-empty string or an integer" + )), + } +} + +pub(crate) fn header_checks( + header: &[(RawKey, RawSqlValue)], + file_path: &str, +) -> Result<(), String> { + if let Some(name) = entry(header, NAME_KEY) + && !matches!(name, RawSqlValue::Str(text) if !python_strip(text).is_empty()) + { + return Err(format!( + "TEST() name in '{file_path}' must be a non-empty string" + )); + } + match entry(header, MODE_KEY) { + None | Some(RawSqlValue::Str(_)) => {} + Some(_) => return Err(format!("TEST() mode in '{file_path}' must be a string")), + } + if let Some(RawSqlValue::Str(mode)) = entry(header, MODE_KEY) + && !SQL_TEST_MODES.contains(&mode.as_str()) + { + return Err(format!( + "TEST() mode in '{file_path}' must be one of: {}", + SQL_TEST_MODES.join(", ") + )); + } + if entry(header, SQL_ANALYSIS_KEY).is_some_and(|value| !matches!(value, RawSqlValue::Bool(_))) { + return Err(format!( + "TEST() sql_analysis in '{file_path}' must be a boolean" + )); + } + Ok(()) +} + +pub(crate) fn test_mode( + header: &[(RawKey, RawSqlValue)], + sql_body: &str, + file_path: &str, +) -> Result<&'static str, String> { + if let Some(RawSqlValue::Str(mode)) = entry(header, MODE_KEY) { + return Ok(SQL_TEST_MODES + .iter() + .find(|known| **known == mode) + .copied() + .unwrap_or(MODEL_MODE)); + } + let modes = defined_modes(sql_body); + if modes.len() > 1 { + return Err(format!( + "SQL test '{file_path}' defines actual CTEs for several test modes ({}); each TEST \ + block tests one kind of logic", + modes.join(", ") + )); + } + Ok(modes[0]) +} diff --git a/crates/sqlbuild-model-config/src/sql_tests/_helpers/mod.rs b/crates/sqlbuild-model-config/src/sql_tests/_helpers/mod.rs new file mode 100644 index 0000000000..76c6432a62 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod header; +pub(crate) mod mode; +pub(crate) mod parameters; diff --git a/crates/sqlbuild-model-config/src/sql_tests/_helpers/mode.rs b/crates/sqlbuild-model-config/src/sql_tests/_helpers/mode.rs new file mode 100644 index 0000000000..2e31c1aa4b --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/_helpers/mode.rs @@ -0,0 +1,127 @@ +//! `_infer_sql_test_mode`: the mode named by a top-level direct-logic actual CTE. + +use crate::sql_tests::constants::{ACTUAL_CTE_MARKER, ACTUAL_CTE_MODES, MODEL_MODE}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::is_python_word::is_python_word; + +fn is_word(character: Option<&char>) -> bool { + character.is_some_and(|character| { + *character == '$' || is_python_word(active_python_text(), *character) + }) +} + +fn find(text: &[char], from: usize, needle: &[char]) -> Option { + (from..=text.len().saturating_sub(needle.len())) + .find(|start| text[*start..].starts_with(needle)) +} + +/// The end of a `quote`-delimited token at `start`, with doubled quotes escaping when `doubled`. +fn quoted_end(text: &[char], start: usize, quote: char, doubled: bool) -> usize { + let mut index = start + 1; + while index < text.len() { + if text[index] == quote { + if doubled && text.get(index + 1) == Some("e) { + index += 2; + continue; + } + return index + 1; + } + index += 1; + } + text.len() +} + +/// The end of a comment or quoted token at `index`, if one starts there. +fn skipped_end(text: &[char], index: usize) -> Option { + match (text[index], text.get(index + 1)) { + ('-', Some('-')) => Some(find(text, index, &['\n']).unwrap_or(text.len())), + ('/', Some('*')) => { + Some(find(text, index + 2, &['*', '/']).map_or(text.len(), |end| end + 2)) + } + ('\'', _) => Some(quoted_end(text, index, '\'', true)), + ('"', _) => Some(quoted_end(text, index, '"', true)), + ('`', _) => Some(quoted_end(text, index, '`', false)), + _ => None, + } +} + +fn skip_space(text: &[char], index: usize) -> usize { + index + + text[index..] + .iter() + .take_while(|character| is_python_space(**character)) + .count() +} + +/// `(?i:AS)(?![\w$])` at `index`. +fn is_as(text: &[char], index: usize) -> bool { + matches!(text.get(index), Some('a' | 'A')) + && matches!(text.get(index + 1), Some('s' | 'S' | '\u{17f}')) + && !is_word(text.get(index + 2)) +} + +/// `(?=\s*(?:\([^()]*\)\s*)?(?i:AS)(?![\w$]))` after a name ending at `end`. +fn defines_cte(text: &[char], end: usize) -> bool { + let after_space = skip_space(text, end); + if text.get(after_space) == Some(&'(') { + let close = text[after_space + 1..] + .iter() + .position(|character| matches!(character, '(' | ')')) + .map(|offset| after_space + 1 + offset); + if let Some(close) = close.filter(|close| text[*close] == ')') + && is_as(text, skip_space(text, close + 1)) + { + return true; + } + } + is_as(text, after_space) +} + +/// The mode of an actual CTE name defined at `index`, and where the name ends. +fn actual_name(text: &[char], index: usize) -> Option<(&'static str, usize)> { + if index > 0 && is_word(text.get(index - 1)) { + return None; + } + ACTUAL_CTE_MODES.iter().find_map(|(name, mode)| { + let name: Vec = name.chars().collect(); + let end = index + name.len(); + (text[index..].starts_with(&name) && !is_word(text.get(end)) && defines_cte(text, end)) + .then_some((*mode, end)) + }) +} + +/// Each mode a top-level actual CTE names, in order of first definition. +pub(crate) fn defined_modes(sql: &str) -> Vec<&'static str> { + if !sql.contains(ACTUAL_CTE_MARKER) { + return vec![MODEL_MODE]; + } + let text: Vec = sql.chars().collect(); + let mut modes: Vec<&'static str> = Vec::new(); + let mut depth: usize = 0; + let mut index = 0; + while index < text.len() { + if let Some(end) = skipped_end(&text, index) { + index = end; + continue; + } + match text[index] { + '(' => depth += 1, + ')' => depth = depth.saturating_sub(1), + _ => { + if let Some((mode, end)) = actual_name(&text, index) { + if depth == 0 && !modes.contains(&mode) { + modes.push(mode); + } + index = end; + continue; + } + } + } + index += 1; + } + if modes.is_empty() { + modes.push(MODEL_MODE); + } + modes +} diff --git a/crates/sqlbuild-model-config/src/sql_tests/_helpers/parameters.rs b/crates/sqlbuild-model-config/src/sql_tests/_helpers/parameters.rs new file mode 100644 index 0000000000..e8f118884f --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/_helpers/parameters.rs @@ -0,0 +1,220 @@ +//! `_parse_test_parameters_and_cases`: declared parameters and their typed case values. + +use crate::sql_tests::constants::{ + CASES_KEY, NULLABLE_OPTION, PARAMETER_TYPES, PARAMETERS_KEY, SQL_VALUE_KINDS, TYPE_OPTION, +}; +use crate::sql_tests::models::{SqlTestCase, SqlTestParameter}; +use crate::sql_values::errors::SqlValueFailure; +use crate::sql_values::main::normalize_sql_value::normalize_sql_value; +use crate::sql_values::models::{RawKey, RawSqlValue, SqlValueLimits}; + +type Entries = [(RawKey, RawSqlValue)]; +type Parsed = Result; + +/// The value of `key` in a header or option mapping. +pub(crate) fn entry<'a>(entries: &'a Entries, key: &str) -> Option<&'a RawSqlValue> { + entries + .iter() + .find(|(candidate, _)| matches!(candidate, RawKey::Str(text) if text == key)) + .map(|(_, value)| value) +} + +fn key_text(key: &RawKey) -> &str { + match key { + RawKey::Str(text) | RawKey::Other(text) => text, + } +} + +/// `^[A-Za-z_][A-Za-z0-9_]*$` under `fullmatch`. +pub(crate) fn is_identifier(text: &str) -> bool { + let mut characters = text.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +fn identifier(key: &RawKey, label: &str, file_path: &str) -> Parsed { + match key { + RawKey::Str(text) if is_identifier(text) => Ok(text.clone()), + _ => Err(format!( + "TEST() {label} name in '{file_path}' must be a SQL identifier" + )), + } +} + +fn non_empty_mapping<'a>( + value: &'a RawSqlValue, + label: &str, + file_path: &str, +) -> Parsed<&'a Entries> { + match value { + RawSqlValue::Object(entries) if !entries.is_empty() => Ok(entries), + _ => Err(format!( + "TEST() {label} in '{file_path}' must be a non-empty mapping" + )), + } +} + +fn declaration_options<'a>( + options: &'a Entries, + name: &str, + file_path: &str, +) -> Parsed<(Option<&'a RawSqlValue>, bool)> { + let mut unknown: Vec<&str> = options + .iter() + .map(|(key, _)| key_text(key)) + .filter(|key| *key != TYPE_OPTION && *key != NULLABLE_OPTION) + .collect(); + unknown.sort_unstable(); + unknown.dedup(); + if !unknown.is_empty() { + return Err(format!( + "TEST() parameter '{name}' in '{file_path}' has unsupported options: {}", + unknown.join(", ") + )); + } + let nullable = match entry(options, NULLABLE_OPTION) { + None => false, + Some(RawSqlValue::Bool(flag)) => *flag, + Some(_) => { + return Err(format!( + "TEST() parameter '{name}' nullable option in '{file_path}' must be boolean" + )); + } + }; + Ok((entry(options, TYPE_OPTION), nullable)) +} + +fn parameter(key: &RawKey, declaration: &RawSqlValue, file_path: &str) -> Parsed { + let name = identifier(key, "parameter", file_path)?; + let (raw_type, nullable) = match declaration { + RawSqlValue::Object(options) => declaration_options(options, &name, file_path)?, + other => (Some(other), false), + }; + let Some(RawSqlValue::Str(raw_type)) = raw_type else { + return Err(format!( + "TEST() parameter '{name}' type in '{file_path}' must be an identifier" + )); + }; + let lowered = raw_type.to_lowercase(); + if !SQL_VALUE_KINDS.contains(&lowered.as_str()) { + return Err(format!( + "TEST() parameter '{name}' in '{file_path}' has unsupported type '{raw_type}'" + )); + } + let Some(value_type) = PARAMETER_TYPES.iter().find(|kind| **kind == lowered) else { + return Err(format!( + "TEST() parameter '{name}' in '{file_path}' must use a scalar type: {}", + PARAMETER_TYPES.join(", ") + )); + }; + Ok(SqlTestParameter { + name, + value_type, + nullable, + }) +} + +fn case_value( + parameter: &SqlTestParameter, + raw: &RawSqlValue, + case_name: &str, + file_path: &str, +) -> Parsed<(String, sqlbuild_core::models::SqlValue)> { + let label = format!( + "TEST() case '{case_name}' parameter '{}' in '{file_path}'", + parameter.name + ); + let is_null = matches!(raw, RawSqlValue::Null); + if is_null && !parameter.nullable { + return Err(format!("{label} is not nullable")); + } + let explicit = (!is_null).then_some(parameter.value_type); + normalize_sql_value(raw, &label, explicit, SqlValueLimits::default()) + .map(|value| (parameter.name.clone(), value)) + .map_err(|failure| match failure { + SqlValueFailure::Invalid(message) => message, + SqlValueFailure::DecimalOverflow => label, + }) +} + +fn case( + (key, raw_values): &(RawKey, RawSqlValue), + parameters: &[SqlTestParameter], + file_path: &str, +) -> Parsed { + let name = identifier(key, "case", file_path)?; + let RawSqlValue::Object(values) = raw_values else { + return Err(format!( + "TEST() case '{name}' in '{file_path}' must define a parameter mapping" + )); + }; + let missing: Vec<&str> = parameters + .iter() + .map(|parameter| parameter.name.as_str()) + .filter(|parameter| entry(values, parameter).is_none()) + .collect(); + let extra: Vec<&str> = values + .iter() + .filter(|(key, _)| { + !matches!(key, RawKey::Str(text) if parameters.iter().any(|parameter| parameter.name == *text)) + }) + .map(|(key, _)| key_text(key)) + .collect(); + if !missing.is_empty() { + return Err(format!( + "TEST() case '{name}' in '{file_path}' is missing parameters: {}", + missing.join(", ") + )); + } + if !extra.is_empty() { + return Err(format!( + "TEST() case '{name}' in '{file_path}' has undeclared parameters: {}", + extra.join(", ") + )); + } + let values = parameters + .iter() + .map(|parameter| { + let raw = entry(values, ¶meter.name).unwrap_or(&RawSqlValue::Null); + case_value(parameter, raw, &name, file_path) + }) + .collect::>>()?; + Ok(SqlTestCase { name, values }) +} + +/// The declared parameters and cases of a TEST header, or why they are invalid. +pub(crate) fn parameters_and_cases( + header: &Entries, + file_path: &str, +) -> Parsed<(Vec, Vec)> { + let (raw_parameters, raw_cases) = + match (entry(header, PARAMETERS_KEY), entry(header, CASES_KEY)) { + (None | Some(RawSqlValue::Null), None | Some(RawSqlValue::Null)) => { + return Ok((Vec::new(), Vec::new())); + } + (Some(parameters), Some(cases)) + if !matches!(parameters, RawSqlValue::Null) + && !matches!(cases, RawSqlValue::Null) => + { + (parameters, cases) + } + _ => { + return Err(format!( + "TEST() in '{file_path}' must define `parameters` and `cases` together" + )); + } + }; + let raw_parameters = non_empty_mapping(raw_parameters, "parameters", file_path)?; + let raw_cases = non_empty_mapping(raw_cases, "cases", file_path)?; + let parameters = raw_parameters + .iter() + .map(|(key, declaration)| parameter(key, declaration, file_path)) + .collect::>>()?; + let cases = raw_cases + .iter() + .map(|raw_case| case(raw_case, ¶meters, file_path)) + .collect::>>()?; + Ok((parameters, cases)) +} diff --git a/crates/sqlbuild-model-config/src/sql_tests/constants.rs b/crates/sqlbuild-model-config/src/sql_tests/constants.rs new file mode 100644 index 0000000000..ff521d4718 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/constants.rs @@ -0,0 +1,28 @@ +//! SQL `TEST(...)` header keys, modes and the actual CTEs that infer a mode. + +pub(crate) const NAME_KEY: &str = "name"; +pub(crate) const MODE_KEY: &str = "mode"; +pub(crate) const SQL_ANALYSIS_KEY: &str = "sql_analysis"; +pub(crate) const PARAMETERS_KEY: &str = "parameters"; +pub(crate) const CASES_KEY: &str = "cases"; +pub(crate) const CURSOR_START_KEY: &str = "cursor_start"; +pub(crate) const CURSOR_END_KEY: &str = "cursor_end"; +pub(crate) const TYPE_OPTION: &str = "type"; +pub(crate) const NULLABLE_OPTION: &str = "nullable"; +/// `SqlTestMode` values, in declaration order. +pub(crate) const SQL_TEST_MODES: [&str; 4] = ["model", "macro", "udf", "table_fn"]; +pub(crate) const MODEL_MODE: &str = "model"; +/// Each direct-logic actual CTE name and the mode it infers, in Python's dict order. +pub(crate) const ACTUAL_CTE_MODES: [(&str, &str); 3] = [ + ("__macro_actual__", "macro"), + ("__udf_actual__", "udf"), + ("__table_fn_actual__", "table_fn"), +]; +/// Text every actual CTE name contains. +pub(crate) const ACTUAL_CTE_MARKER: &str = "_actual__"; +/// The scalar kinds a test parameter may declare, in `SqlValueKind` order. +pub(crate) const PARAMETER_TYPES: [&str; 5] = ["string", "integer", "boolean", "float", "decimal"]; +/// Every `SqlValueKind` value. +pub(crate) const SQL_VALUE_KINDS: [&str; 9] = [ + "string", "integer", "boolean", "float", "decimal", "null", "list", "set", "object", +]; diff --git a/crates/sqlbuild-model-config/src/sql_tests/main/mod.rs b/crates/sqlbuild-model-config/src/sql_tests/main/mod.rs new file mode 100644 index 0000000000..1a59974290 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/main/mod.rs @@ -0,0 +1,2 @@ +pub mod sql_test_block; +pub mod sql_test_names_failure; diff --git a/crates/sqlbuild-model-config/src/sql_tests/main/sql_test_block.rs b/crates/sqlbuild-model-config/src/sql_tests/main/sql_test_block.rs new file mode 100644 index 0000000000..99ada140f3 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/main/sql_test_block.rs @@ -0,0 +1,43 @@ +//! Python's `build_sql_test_block`: validate one TEST block's header against its SQL. + +use crate::sql_tests::_helpers::header::{cursor_bound, header_checks, test_mode}; +use crate::sql_tests::_helpers::parameters::{entry, parameters_and_cases}; +use crate::sql_tests::constants::{CURSOR_END_KEY, CURSOR_START_KEY, MODEL_MODE, NAME_KEY}; +use crate::sql_tests::models::SqlTestBlock; +use crate::sql_values::models::{RawKey, RawSqlValue}; + +/// The validated fields of one TEST block, or the `SqlTestParseError` message rejecting it. +pub fn sql_test_block( + header: &[(RawKey, RawSqlValue)], + sql_body: &str, + file_path: &str, +) -> Result { + header_checks(header, file_path)?; + if sql_body.is_empty() { + return Err(format!( + "SQL test '{file_path}' must define SQL after TEST(...)" + )); + } + let name = match entry(header, NAME_KEY) { + Some(RawSqlValue::Str(text)) => Some(text.clone()), + _ => None, + }; + let mode = test_mode(header, sql_body, file_path)?; + let (parameters, cases) = parameters_and_cases(header, file_path)?; + let cursor_start = cursor_bound(header, CURSOR_START_KEY, file_path)?; + let cursor_end = cursor_bound(header, CURSOR_END_KEY, file_path)?; + if (cursor_start.is_some() || cursor_end.is_some()) && mode != MODEL_MODE { + return Err(format!( + "TEST() in '{file_path}' declares `cursor_start` or `cursor_end`, which are only \ + supported for model tests, not mode '{mode}'" + )); + } + Ok(SqlTestBlock { + name, + mode, + parameters, + cases, + cursor_start, + cursor_end, + }) +} diff --git a/crates/sqlbuild-model-config/src/sql_tests/main/sql_test_names_failure.rs b/crates/sqlbuild-model-config/src/sql_tests/main/sql_test_names_failure.rs new file mode 100644 index 0000000000..a7c1f91b2e --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/main/sql_test_names_failure.rs @@ -0,0 +1,33 @@ +//! Python's `validate_sql_test_names`: every block of a multi-block file has a unique name. + +/// Why the blocks' `(test_index, name)` pairs are invalid, if they are. +pub fn sql_test_names_failure( + file_path: &str, + blocks: &[(usize, Option)], +) -> Option { + if blocks.len() <= 1 { + return None; + } + let unnamed: Vec = blocks + .iter() + .filter(|(_, name)| name.is_none()) + .map(|(index, _)| index.to_string()) + .collect(); + if !unnamed.is_empty() { + return Some(format!( + "SQL test '{file_path}' contains multiple TEST blocks; every block must define a \ + unique `name`. Missing names for blocks: {}", + unnamed.join(", ") + )); + } + let mut seen: Vec<&str> = Vec::new(); + for name in blocks.iter().filter_map(|(_, name)| name.as_deref()) { + if seen.contains(&name) { + return Some(format!( + "SQL test '{file_path}' defines duplicate TEST() name '{name}'" + )); + } + seen.push(name); + } + None +} diff --git a/crates/sqlbuild-model-config/src/sql_tests/mod.rs b/crates/sqlbuild-model-config/src/sql_tests/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/sql_tests/models.rs b/crates/sqlbuild-model-config/src/sql_tests/models.rs new file mode 100644 index 0000000000..f8b064e738 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/models.rs @@ -0,0 +1,29 @@ +//! A validated SQL `TEST(...)` block. + +use sqlbuild_core::models::SqlValue; + +/// One declared parameter: name, `SqlValueKind` value and nullability. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SqlTestParameter { + pub name: String, + pub value_type: &'static str, + pub nullable: bool, +} + +/// One named case and its typed parameter values, in declaration order. +#[derive(Clone, Debug, PartialEq)] +pub struct SqlTestCase { + pub name: String, + pub values: Vec<(String, SqlValue)>, +} + +/// The fields `build_sql_test_block` derives from one TEST block's header and body. +#[derive(Clone, Debug, PartialEq)] +pub struct SqlTestBlock { + pub name: Option, + pub mode: &'static str, + pub parameters: Vec, + pub cases: Vec, + pub cursor_start: Option, + pub cursor_end: Option, +} diff --git a/crates/sqlbuild-model-config/src/sql_tests/tests.rs b/crates/sqlbuild-model-config/src/sql_tests/tests.rs new file mode 100644 index 0000000000..5d59acf4cd --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_sql_test_block.rs"] +mod sql_test_block; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/sql_tests/tests/test_sql_test_block.rs b/crates/sqlbuild-model-config/src/sql_tests/tests/test_sql_test_block.rs new file mode 100644 index 0000000000..f62071dc94 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/tests/test_sql_test_block.rs @@ -0,0 +1,43 @@ +use crate::sql_tests::main::sql_test_block::sql_test_block; +use crate::sql_tests::tests::test_types::SqlTestModeTestCase; + +#[test] +fn given_test_bodies_when_inferring_mode_then_top_level_actual_ctes_decide() { + let test_cases = [ + SqlTestModeTestCase { + description: "plain SQL tests a model", + sql_body: "SELECT 1", + expected_mode: Ok("model"), + }, + SqlTestModeTestCase { + description: "a top-level macro actual CTE with a column list", + sql_body: "WITH __macro_actual__ (a) AS (SELECT 1) SELECT 1", + expected_mode: Ok("macro"), + }, + SqlTestModeTestCase { + description: "actual CTE names in strings, comments and subqueries are ignored", + sql_body: "SELECT '__udf_actual__ AS', (WITH __udf_actual__ AS (SELECT 1) SELECT 1) \ + -- __macro_actual__ AS", + expected_mode: Ok("model"), + }, + SqlTestModeTestCase { + description: "two kinds of actual CTE are rejected", + sql_body: "WITH __udf_actual__ AS (SELECT 1), __macro_actual__ AS (SELECT 2) SELECT 1", + expected_mode: Err( + "SQL test 't.sql' defines actual CTEs for several test modes (udf, macro); each \ + TEST block tests one kind of logic", + ), + }, + ]; + + for test_case in test_cases { + let mode = sql_test_block(&[], test_case.sql_body, "t.sql").map(|block| block.mode); + + assert_eq!( + mode, + test_case.expected_mode.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/sql_tests/tests/test_types.rs b/crates/sqlbuild-model-config/src/sql_tests/tests/test_types.rs new file mode 100644 index 0000000000..c2accfa8f0 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_tests/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(super) struct SqlTestModeTestCase { + pub(super) description: &'static str, + pub(super) sql_body: &'static str, + pub(super) expected_mode: Result<&'static str, &'static str>, +} diff --git a/crates/sqlbuild-model-config/src/sql_values/_helpers/decimals.rs b/crates/sqlbuild-model-config/src/sql_values/_helpers/decimals.rs new file mode 100644 index 0000000000..67a0e0412e --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/_helpers/decimals.rs @@ -0,0 +1,235 @@ +//! Python's `Decimal(text)`, `str(Decimal)` and `Decimal.normalize().as_tuple()`. + +use crate::sql_values::constants::{ + DECIMAL_EMAX, DECIMAL_EMIN, DECIMAL_MAX_EXPONENT, DECIMAL_PRECISION, PLAIN_NOTATION_FLOOR, +}; +use crate::sql_values::errors::SqlValueFailure; +use crate::sql_values::models::{DecimalNumber, DecimalSpecial}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_decimal_value::python_decimal_value; +use sqlbuild_core::text::main::python_strip::python_strip; + +const ZERO_DIGITS: &str = "0"; +const INFINITY_SPELLINGS: [&str; 2] = ["inf", "infinity"]; +const NAN_PREFIX: &str = "nan"; +const SIGNALING_NAN_PREFIX: &str = "snan"; +/// libmpdec's smallest exponent: `MIN_EMIN - (MAX_PREC - 1)`. +const DECIMAL_MIN_EXPONENT: i128 = -1_999_999_999_999_999_997; + +/// The text with whitespace stripped, underscores dropped and Unicode digits made ASCII. +fn ascii_text(text: &str) -> Option { + let python = active_python_text(); + python_strip(text) + .chars() + .filter(|character| *character != '_') + .map(|character| { + if character.is_ascii() { + return Some(character.to_ascii_lowercase()); + } + python_decimal_value(python, character).and_then(|value| char::from_digit(value, 10)) + }) + .collect() +} + +fn without_leading_zeros(digits: &str) -> String { + let trimmed = digits.trim_start_matches('0'); + if trimmed.is_empty() { + "0".to_owned() + } else { + trimmed.to_owned() + } +} + +fn is_digits(text: &str) -> bool { + !text.is_empty() && text.bytes().all(|byte| byte.is_ascii_digit()) +} + +fn special(body: &str) -> Option { + if INFINITY_SPELLINGS.contains(&body) { + return Some(DecimalSpecial::Infinity); + } + let payload = |rest: &str| { + (rest.is_empty() || is_digits(rest)).then(|| { + let digits = without_leading_zeros(rest); + if digits == ZERO_DIGITS { + String::new() + } else { + digits + } + }) + }; + if let Some(rest) = body.strip_prefix(SIGNALING_NAN_PREFIX) { + return payload(rest).map(DecimalSpecial::SignalingNaN); + } + body.strip_prefix(NAN_PREFIX) + .and_then(payload) + .map(DecimalSpecial::NaN) +} + +fn exponent_value(text: Option<&str>) -> Option { + let Some(text) = text else { + return Some(0); + }; + let (negative, digits) = match text.as_bytes().first() { + Some(b'-') => (true, &text[1..]), + Some(b'+') => (false, &text[1..]), + _ => (false, text), + }; + if !is_digits(digits) { + return None; + } + let magnitude = without_leading_zeros(digits); + let value = magnitude.parse::().unwrap_or(i128::MAX); + Some(if negative { -value } else { value }) +} + +/// `Decimal(text)`, or `None` where Python raises `InvalidOperation`. +pub(crate) fn parse_decimal(text: &str) -> Option { + let ascii = ascii_text(text)?; + let (negative, body) = match ascii.as_bytes().first() { + Some(b'-') => (true, &ascii[1..]), + Some(b'+') => (false, &ascii[1..]), + _ => (false, ascii.as_str()), + }; + if let Some(special) = special(body) { + return Some(DecimalNumber { + negative, + special: Some(special), + digits: "0".to_owned(), + exponent: 0, + }); + } + let (mantissa, exponent) = match body.split_once('e') { + Some((mantissa, exponent)) => (mantissa, Some(exponent)), + None => (body, None), + }; + let (whole, fraction) = mantissa.split_once('.').unwrap_or((mantissa, "")); + let digits_ok = + (whole.is_empty() || is_digits(whole)) && (fraction.is_empty() || is_digits(fraction)); + if !digits_ok || (whole.is_empty() && fraction.is_empty()) { + return None; + } + let exponent = exponent_value(exponent)?.saturating_sub(fraction.len() as i128); + if exponent > i128::from(DECIMAL_MAX_EXPONENT) || exponent < DECIMAL_MIN_EXPONENT { + return None; + } + Some(DecimalNumber { + negative, + special: None, + digits: without_leading_zeros(&format!("{whole}{fraction}")), + exponent: exponent as i64, + }) +} + +/// `str(Decimal)`: the scientific string of the decimal specification. +pub(crate) fn decimal_text(number: &DecimalNumber) -> String { + let sign = if number.negative { "-" } else { "" }; + match &number.special { + Some(DecimalSpecial::Infinity) => return format!("{sign}Infinity"), + Some(DecimalSpecial::NaN(payload)) => return format!("{sign}NaN{payload}"), + Some(DecimalSpecial::SignalingNaN(payload)) => return format!("{sign}sNaN{payload}"), + None => {} + } + let digits = &number.digits; + let length = digits.len() as i64; + let left_digits = number.exponent + length; + let dot_place = if number.exponent <= 0 && left_digits > PLAIN_NOTATION_FLOOR { + left_digits + } else { + 1 + }; + let (whole, fraction) = if dot_place <= 0 { + ( + "0".to_owned(), + format!(".{}{digits}", "0".repeat((-dot_place) as usize)), + ) + } else if dot_place >= length { + ( + format!("{digits}{}", "0".repeat((dot_place - length) as usize)), + String::new(), + ) + } else { + let split = dot_place as usize; + (digits[..split].to_owned(), format!(".{}", &digits[split..])) + }; + let exponent = if left_digits == dot_place { + String::new() + } else { + format!("E{:+}", left_digits - dot_place) + }; + format!("{sign}{whole}{fraction}{exponent}") +} + +/// `digits` rounded half-even to drop its last `drop` digits, and whether that carried. +fn round_half_even(digits: &str, drop: usize) -> String { + if drop == 0 { + return digits.to_owned(); + } + if drop > digits.len() { + return "0".to_owned(); + } + let (kept, dropped) = digits.split_at(digits.len() - drop); + let first = dropped.as_bytes()[0]; + let rest_nonzero = dropped[1..].bytes().any(|byte| byte != b'0'); + let last_odd = kept + .as_bytes() + .last() + .is_some_and(|byte| (byte - b'0') % 2 == 1); + let round_up = first > b'5' || (first == b'5' && (rest_nonzero || last_odd)); + let mut values: Vec = if kept.is_empty() { + vec![b'0'] + } else { + kept.bytes().collect() + }; + if round_up { + let mut index = values.len(); + loop { + if index == 0 { + values.insert(0, b'1'); + break; + } + index -= 1; + if values[index] == b'9' { + values[index] = b'0'; + } else { + values[index] += 1; + break; + } + } + } + without_leading_zeros(&String::from_utf8(values).unwrap_or_default()) +} + +/// `Decimal.normalize().as_tuple()` in the default context: sign, digits and exponent. +pub(crate) fn normalized_parts( + number: &DecimalNumber, +) -> Result<(bool, String, i64), SqlValueFailure> { + if number.digits == ZERO_DIGITS { + return Ok((number.negative, "0".to_owned(), 0)); + } + let mut digits = number.digits.clone(); + let mut exponent = number.exponent; + if digits.len() > DECIMAL_PRECISION { + let drop = digits.len() - DECIMAL_PRECISION; + digits = round_half_even(&digits, drop); + exponent += drop as i64; + if digits.len() > DECIMAL_PRECISION { + digits.pop(); + exponent += 1; + } + } + if exponent + digits.len() as i64 - 1 > DECIMAL_EMAX { + return Err(SqlValueFailure::DecimalOverflow); + } + let tiny = DECIMAL_EMIN - (DECIMAL_PRECISION as i64 - 1); + if exponent < tiny { + digits = round_half_even(&digits, (tiny - exponent) as usize); + exponent = tiny; + if digits == ZERO_DIGITS { + return Ok((number.negative, digits, 0)); + } + } + let trimmed = digits.trim_end_matches('0'); + exponent += (digits.len() - trimmed.len()) as i64; + Ok((number.negative, trimmed.to_owned(), exponent)) +} diff --git a/crates/sqlbuild-model-config/src/sql_values/_helpers/mod.rs b/crates/sqlbuild-model-config/src/sql_values/_helpers/mod.rs new file mode 100644 index 0000000000..b6174dc60e --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod decimals; +pub(crate) mod normalize; +pub(crate) mod reprs; diff --git a/crates/sqlbuild-model-config/src/sql_values/_helpers/normalize.rs b/crates/sqlbuild-model-config/src/sql_values/_helpers/normalize.rs new file mode 100644 index 0000000000..3ad37ec319 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/_helpers/normalize.rs @@ -0,0 +1,364 @@ +//! The recursion of `normalize_sql_value`, with Python's accounting and diagnostics. + +use std::cell::Cell; + +use crate::sql_values::_helpers::decimals::{decimal_text, parse_decimal}; +use crate::sql_values::_helpers::reprs::{identity_repr, payload_repr}; +use crate::sql_values::constants::{ + BOOLEAN_SIZE, DECIMAL_KIND, DELIMITER_SIZE, NULL_SIZE, POSITIVE_ZERO, SCALAR_KINDS, +}; +use crate::sql_values::errors::SqlValueFailure; +use crate::sql_values::models::{RawKey, RawSqlValue, SqlValueLimits}; +use sqlbuild_core::json::main::python_float_repr::python_float_repr; +use sqlbuild_core::models::{SqlLogicalType, SqlValue, SqlValueKind}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +type Normalized = Result; + +/// The running totals one normalization accounts against its limits. +pub(crate) struct Normalization<'a> { + pub(crate) context: &'a str, + pub(crate) limits: SqlValueLimits, + pub(crate) elements: Cell, + pub(crate) size: Cell, +} + +/// One value's place: its diagnostic path and nesting depth. +#[derive(Clone, Copy)] +pub(crate) struct Place<'p> { + pub(crate) path: &'p str, + pub(crate) depth: i64, +} + +fn invalid(message: String) -> SqlValueFailure { + SqlValueFailure::Invalid(message) +} + +impl Normalization<'_> { + fn fail(&self, path: &str, problem: &str) -> SqlValueFailure { + invalid(format!("{}{path} {problem}", self.context)) + } + + fn account(&self, path: &str, (elements, size): (i64, i64)) -> Result<(), SqlValueFailure> { + self.elements.set(self.elements.get() + elements); + self.size.set(self.size.get() + size); + if self.elements.get() > self.limits.max_elements { + return Err(self.fail( + path, + &format!( + "exceeds the maximum element count of {}", + self.limits.max_elements + ), + )); + } + if self.size.get() > self.limits.max_size { + return Err(self.fail( + path, + &format!( + "exceeds the maximum value size of {} bytes", + self.limits.max_size + ), + )); + } + Ok(()) + } +} + +fn text_size(text: &str) -> i64 { + text.len() as i64 + DELIMITER_SIZE +} + +fn type_name(raw: &RawSqlValue) -> &str { + match raw { + RawSqlValue::Null => "NoneType", + RawSqlValue::Bool(_) => "bool", + RawSqlValue::Int(_) | RawSqlValue::BigInt(_) => "int", + RawSqlValue::Float(_) => "float", + RawSqlValue::Decimal(_) => "Decimal", + RawSqlValue::Str(_) => "str", + RawSqlValue::List(_) => "list", + RawSqlValue::Set(_) => "AuthoredSqlSet", + RawSqlValue::Object(_) => "dict", + RawSqlValue::Call(_) => "AuthoredSqlValueCall", + RawSqlValue::Other(name) => name, + } +} + +fn scalar_decimal(state: &Normalization<'_>, path: &str, text: &str) -> Normalized { + let number = parse_decimal(text).ok_or_else(|| state.fail(path, "has an invalid decimal"))?; + if number.special.is_some() { + return Err(state.fail( + path, + &format!("decimal must be finite; received Decimal('{text}')"), + )); + } + state.account(path, (0, text.chars().count() as i64))?; + Ok(SqlValue::Decimal(decimal_text(&number))) +} + +fn scalar(raw: &RawSqlValue, state: &Normalization<'_>, path: &str) -> Option { + Some(match raw { + RawSqlValue::Null => state.account(path, (0, NULL_SIZE)).map(|()| SqlValue::Null), + RawSqlValue::Bool(flag) => state + .account(path, (0, BOOLEAN_SIZE)) + .map(|()| SqlValue::Boolean(*flag)), + RawSqlValue::Int(number) => state + .account(path, (0, number.to_string().len() as i64)) + .map(|()| SqlValue::Integer(*number)), + RawSqlValue::BigInt(digits) => Err(state.fail( + path, + &format!("integer {digits} is outside the signed 64-bit range"), + )), + RawSqlValue::Float(number) if !number.is_finite() => Err(state.fail( + path, + &format!( + "float must be finite; received {}", + python_float_repr(*number) + ), + )), + RawSqlValue::Float(number) => { + let normalized = if *number == POSITIVE_ZERO { + POSITIVE_ZERO + } else { + *number + }; + state + .account(path, (0, python_float_repr(normalized).len() as i64)) + .map(|()| SqlValue::Float(normalized)) + } + RawSqlValue::Decimal(text) => scalar_decimal(state, path, text), + RawSqlValue::Str(text) => state + .account(path, (0, text_size(text))) + .map(|()| SqlValue::String(text.clone())), + _ => return None, + }) +} + +/// `_normalize`: one authored value at `place`, typed by `explicit_type` when given. +pub(crate) fn normalize( + raw: &RawSqlValue, + place: Place<'_>, + state: &Normalization<'_>, + explicit_type: Option<&str>, +) -> Normalized { + if place.depth > state.limits.max_depth { + return Err(state.fail( + place.path, + &format!( + "exceeds the maximum nesting depth of {}", + state.limits.max_depth + ), + )); + } + if let Some(explicit) = explicit_type { + return explicit_scalar(raw, place.path, state, explicit); + } + if let Some(normalized) = scalar(raw, state, place.path) { + return normalized; + } + match raw { + RawSqlValue::List(items) => collection(items, SqlValueKind::List, place, state), + RawSqlValue::Set(items) => collection(items, SqlValueKind::Set, place, state), + RawSqlValue::Object(entries) => object(entries, place, state), + _ => Err(state.fail( + place.path, + &format!( + "has unsupported type {}; expected a scalar, list, set, or object", + type_name(raw) + ), + )), + } +} + +fn explicit_decimal(raw: &RawSqlValue, path: &str, state: &Normalization<'_>) -> Normalized { + let RawSqlValue::Str(text) = raw else { + return Err(state.fail( + path, + &format!( + "with type decimal requires a quoted decimal string; received {}", + type_name(raw) + ), + )); + }; + let quoted = python_str_repr(active_python_text(), text); + let number = parse_decimal(text) + .ok_or_else(|| state.fail(path, &format!("has invalid decimal value {quoted}")))?; + if number.special.is_some() { + return Err(state.fail(path, &format!("decimal must be finite; received {quoted}"))); + } + state.account(path, (0, text.chars().count() as i64))?; + Ok(SqlValue::Decimal(decimal_text(&number))) +} + +fn explicit_scalar( + raw: &RawSqlValue, + path: &str, + state: &Normalization<'_>, + explicit: &str, +) -> Normalized { + let kind = explicit.to_lowercase(); + let known = + ["list", "set", "object"].contains(&kind.as_str()) || SCALAR_KINDS.contains(&kind.as_str()); + if !known { + return Err(state.fail( + path, + &format!( + "has unsupported explicit type '{explicit}'; expected {}", + SCALAR_KINDS.join(", ") + ), + )); + } + if !SCALAR_KINDS.contains(&kind.as_str()) { + return Err(state.fail( + path, + &format!("explicit type '{explicit}' is not a scalar type"), + )); + } + if kind == DECIMAL_KIND { + return explicit_decimal(raw, path, state); + } + let matches = matches!( + (kind.as_str(), raw), + ("string", RawSqlValue::Str(_)) + | ("integer", RawSqlValue::Int(_) | RawSqlValue::BigInt(_)) + | ("boolean", RawSqlValue::Bool(_)) + | ("float", RawSqlValue::Float(_)) + | ("null", RawSqlValue::Null) + ); + if !matches { + return Err(state.fail( + path, + &format!("has type {}; expected {kind}", type_name(raw)), + )); + } + normalize(raw, Place { path, depth: 1 }, state, None) +} + +fn element_type( + values: &[SqlValue], + path: &str, + kind: SqlValueKind, + state: &Normalization<'_>, +) -> Result { + let expected = values + .iter() + .find(|value| !matches!(value, SqlValue::Null)) + .map(SqlValue::logical_type) + .ok_or_else(|| { + state.fail( + path, + &format!( + "{} cannot infer an element type from only null values", + kind.as_str() + ), + ) + })?; + for (index, value) in values.iter().enumerate() { + let logical = value.logical_type(); + if !matches!(value, SqlValue::Null) && logical != expected { + return Err(state.fail( + &format!("{path}[{index}]"), + &format!( + "has type {}; expected {}", + logical.display_name(), + expected.display_name() + ), + )); + } + } + Ok(expected) +} + +fn canonical_set( + values: Vec, + path: &str, + state: &Normalization<'_>, +) -> Result, SqlValueFailure> { + let mut keyed: Vec<(String, SqlValue)> = Vec::with_capacity(values.len()); + for (index, value) in values.into_iter().enumerate() { + let identity = identity_repr(&value)?; + if keyed.iter().any(|(seen, _)| *seen == identity) { + return Err(state.fail( + path, + &format!( + "contains duplicate set value {} at {path}[{index}]", + payload_repr(&value) + ), + )); + } + keyed.push((identity, value)); + } + keyed.sort_by(|(left, _), (right, _)| left.cmp(right)); + Ok(keyed.into_iter().map(|(_, value)| value).collect()) +} + +fn collection( + items: &[RawSqlValue], + kind: SqlValueKind, + place: Place<'_>, + state: &Normalization<'_>, +) -> Normalized { + let path = place.path; + if items.is_empty() { + return Err(state.fail( + path, + &format!("{} must contain at least one value", kind.as_str()), + )); + } + let count = items.len() as i64; + state.account(path, (count, DELIMITER_SIZE + count))?; + let mut values: Vec = Vec::with_capacity(items.len()); + for (index, item) in items.iter().enumerate() { + let item_path = format!("{path}[{index}]"); + let item_place = Place { + path: &item_path, + depth: place.depth + 1, + }; + values.push(normalize(item, item_place, state, None)?); + } + let element = element_type(&values, path, kind, state)?; + if kind == SqlValueKind::Set { + let values = canonical_set(values, path, state)?; + return Ok(SqlValue::Set { + element_type: element, + values, + }); + } + Ok(SqlValue::List { + element_type: element, + values, + }) +} + +fn object( + entries: &[(RawKey, RawSqlValue)], + place: Place<'_>, + state: &Normalization<'_>, +) -> Normalized { + let path = place.path; + let count = entries.len() as i64; + state.account(path, (count, DELIMITER_SIZE + count))?; + let mut normalized: Vec<(String, SqlValue)> = Vec::with_capacity(entries.len()); + for (key, item) in entries { + let key = match key { + RawKey::Str(key) => key, + RawKey::Other(repr) => { + return Err(state.fail(path, &format!("object key {repr} must be a string"))); + } + }; + state.account(path, (0, text_size(key)))?; + let item_path = if path.is_empty() { + format!(".{key}") + } else { + format!("{path}.{key}") + }; + let item_place = Place { + path: &item_path, + depth: place.depth + 1, + }; + normalized.push((key.clone(), normalize(item, item_place, state, None)?)); + } + normalized.sort_by(|(left, _), (right, _)| left.cmp(right)); + Ok(SqlValue::Object(normalized)) +} diff --git a/crates/sqlbuild-model-config/src/sql_values/_helpers/reprs.rs b/crates/sqlbuild-model-config/src/sql_values/_helpers/reprs.rs new file mode 100644 index 0000000000..f4ccfd3b25 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/_helpers/reprs.rs @@ -0,0 +1,102 @@ +//! Python reprs of normalized values: `sql_value_identity` keys and `SqlValue` payloads. + +use crate::sql_values::_helpers::decimals::{normalized_parts, parse_decimal}; +use crate::sql_values::errors::SqlValueFailure; +use sqlbuild_core::json::main::python_float_repr::python_float_repr; +use sqlbuild_core::models::{SqlLogicalType, SqlValue}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +fn text_repr(text: &str) -> String { + python_str_repr(active_python_text(), text) +} + +/// `repr(tuple(items))`, with the trailing comma of a one-item tuple. +fn tuple_repr(items: &[String]) -> String { + match items { + [only] => format!("({only},)"), + _ => format!("({})", items.join(", ")), + } +} + +fn bool_repr(flag: bool) -> &'static str { + if flag { "True" } else { "False" } +} + +fn decimal_tuple_repr(text: &str) -> Result { + let number = parse_decimal(text).ok_or_else(|| SqlValueFailure::Invalid(text.to_owned()))?; + let (negative, digits, exponent) = normalized_parts(&number)?; + let digits: Vec = digits.chars().map(String::from).collect(); + Ok(format!( + "DecimalTuple(sign={}, digits={}, exponent={exponent})", + u8::from(negative), + tuple_repr(&digits) + )) +} + +/// `repr(sql_value_identity(value=value))`; the text orders and deduplicates set members. +pub(crate) fn identity_repr(value: &SqlValue) -> Result { + let kind = value.logical_type().kind.as_str(); + let payload = match value { + SqlValue::Null => "None".to_owned(), + SqlValue::Boolean(flag) => bool_repr(*flag).to_owned(), + SqlValue::Integer(number) => number.to_string(), + SqlValue::Float(number) => python_float_repr(*number), + SqlValue::Decimal(text) => decimal_tuple_repr(text)?, + SqlValue::String(text) => text_repr(text), + SqlValue::List { values, .. } | SqlValue::Set { values, .. } => tuple_repr( + &values + .iter() + .map(identity_repr) + .collect::, _>>()?, + ), + SqlValue::Object(entries) => tuple_repr( + &entries + .iter() + .map(|(key, item)| Ok(format!("({}, {})", text_repr(key), identity_repr(item)?))) + .collect::, SqlValueFailure>>()?, + ), + }; + Ok(format!("('{kind}', {payload})")) +} + +fn logical_repr(logical: &SqlLogicalType) -> String { + let kind = logical.kind.as_str(); + let element = logical + .element_type + .as_deref() + .map_or_else(|| "None".to_owned(), logical_repr); + format!( + "SqlLogicalType(kind=, element_type={element})", + kind.to_ascii_uppercase() + ) +} + +fn sql_value_repr(value: &SqlValue) -> String { + format!( + "SqlValue(logical_type={}, value={})", + logical_repr(&value.logical_type()), + payload_repr(value) + ) +} + +/// `repr(value.value)`. +pub(crate) fn payload_repr(value: &SqlValue) -> String { + match value { + SqlValue::Null => "None".to_owned(), + SqlValue::Boolean(flag) => bool_repr(*flag).to_owned(), + SqlValue::Integer(number) => number.to_string(), + SqlValue::Float(number) => python_float_repr(*number), + SqlValue::Decimal(text) => format!("Decimal('{text}')"), + SqlValue::String(text) => text_repr(text), + SqlValue::List { values, .. } | SqlValue::Set { values, .. } => { + tuple_repr(&values.iter().map(sql_value_repr).collect::>()) + } + SqlValue::Object(entries) => tuple_repr( + &entries + .iter() + .map(|(key, item)| format!("({}, {})", text_repr(key), sql_value_repr(item))) + .collect::>(), + ), + } +} diff --git a/crates/sqlbuild-model-config/src/sql_values/constants.rs b/crates/sqlbuild-model-config/src/sql_values/constants.rs new file mode 100644 index 0000000000..fb46df6b2c --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/constants.rs @@ -0,0 +1,22 @@ +//! Typed SQL value limits and the Python spellings their diagnostics use. + +/// `decimal.DefaultContext.prec`, which `Decimal.normalize()` rounds to. +pub(crate) const DECIMAL_PRECISION: usize = 28; +/// `decimal.DefaultContext.Emax`. +pub(crate) const DECIMAL_EMAX: i64 = 999_999; +/// `decimal.DefaultContext.Emin`. +pub(crate) const DECIMAL_EMIN: i64 = -999_999; +/// Python's `MAX_EMAX`: larger exponents make `Decimal(text)` invalid. +pub(crate) const DECIMAL_MAX_EXPONENT: i64 = 999_999_999_999_999_999; +/// Plain notation is used down to this many places before the first digit. +pub(crate) const PLAIN_NOTATION_FLOOR: i64 = -6; +/// The scalar kinds, in `SqlValueKind` order. +pub(crate) const SCALAR_KINDS: [&str; 6] = + ["string", "integer", "boolean", "float", "decimal", "null"]; +pub(crate) const NULL_SIZE: i64 = 4; +pub(crate) const BOOLEAN_SIZE: i64 = 5; +/// Quotes or brackets around a string, key, or collection. +pub(crate) const DELIMITER_SIZE: i64 = 2; +pub(crate) const DECIMAL_KIND: &str = "decimal"; +/// `0.0`, which also stands in for a normalized `-0.0`. +pub(crate) const POSITIVE_ZERO: f64 = 0.0; diff --git a/crates/sqlbuild-model-config/src/sql_values/errors.rs b/crates/sqlbuild-model-config/src/sql_values/errors.rs new file mode 100644 index 0000000000..9015ac9c50 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/errors.rs @@ -0,0 +1,10 @@ +//! Why a typed SQL value is rejected. + +/// The exception normalizing a value raises. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SqlValueFailure { + /// `SqlValueValidationError` with this message. + Invalid(String), + /// `decimal.Overflow`, which `Decimal.normalize()` raises for an exponent past `Emax`. + DecimalOverflow, +} diff --git a/crates/sqlbuild-model-config/src/sql_values/main/mod.rs b/crates/sqlbuild-model-config/src/sql_values/main/mod.rs new file mode 100644 index 0000000000..9106bb25c9 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/main/mod.rs @@ -0,0 +1 @@ +pub mod normalize_sql_value; diff --git a/crates/sqlbuild-model-config/src/sql_values/main/normalize_sql_value.rs b/crates/sqlbuild-model-config/src/sql_values/main/normalize_sql_value.rs new file mode 100644 index 0000000000..cbd2791104 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/main/normalize_sql_value.rs @@ -0,0 +1,24 @@ +//! Python's `normalize_sql_value`: validate one authored value into its canonical typed form. + +use std::cell::Cell; + +use crate::sql_values::_helpers::normalize::{Normalization, Place, normalize}; +use crate::sql_values::errors::SqlValueFailure; +use crate::sql_values::models::{RawSqlValue, SqlValueLimits}; +use sqlbuild_core::models::SqlValue; + +/// The canonical value of `raw`, typed by `explicit_type` when given, under `limits`. +pub fn normalize_sql_value( + raw: &RawSqlValue, + context: &str, + explicit_type: Option<&str>, + limits: SqlValueLimits, +) -> Result { + let state = Normalization { + context, + limits, + elements: Cell::new(0), + size: Cell::new(0), + }; + normalize(raw, Place { path: "", depth: 1 }, &state, explicit_type) +} diff --git a/crates/sqlbuild-model-config/src/sql_values/mod.rs b/crates/sqlbuild-model-config/src/sql_values/mod.rs new file mode 100644 index 0000000000..383898ca23 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-model-config/src/sql_values/models.rs b/crates/sqlbuild-model-config/src/sql_values/models.rs new file mode 100644 index 0000000000..30f2eac6d6 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/models.rs @@ -0,0 +1,67 @@ +//! Authored values as normalization reads them. + +/// An authored value: the Python objects `normalize_sql_value` accepts, by exact type. +#[derive(Clone, Debug, PartialEq)] +pub enum RawSqlValue { + Null, + Bool(bool), + /// An `int` in the signed 64-bit range. + Int(i64), + /// Any other `int`, as `str(value)`. + BigInt(String), + Float(f64), + /// A `Decimal`, as `str(value)`. + Decimal(String), + Str(String), + List(Vec), + /// An `AuthoredSqlSet`. + Set(Vec), + Object(Vec<(RawKey, RawSqlValue)>), + /// An `AuthoredSqlValueCall`: the `constant(...)` arguments in authored order. + Call(Vec<(String, RawSqlValue)>), + /// Any other object, by `type(value).__name__`. + Other(String), +} + +/// An object key: a `str`, or any other object by its `repr`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RawKey { + Str(String), + Other(String), +} + +/// `SqlValueLimits`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SqlValueLimits { + pub max_depth: i64, + pub max_elements: i64, + pub max_size: i64, +} + +impl Default for SqlValueLimits { + fn default() -> Self { + Self { + max_depth: 32, + max_elements: 10_000, + max_size: 1_000_000, + } + } +} + +/// A parsed finite or special `Decimal`: sign, coefficient digits and exponent. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct DecimalNumber { + pub(crate) negative: bool, + pub(crate) special: Option, + /// Coefficient digits without leading zeros; `"0"` for zero. + pub(crate) digits: String, + pub(crate) exponent: i64, +} + +/// `Infinity`, `NaN` or `sNaN` with its diagnostic payload digits. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum DecimalSpecial { + Infinity, + NaN(String), + SignalingNaN(String), +} diff --git a/crates/sqlbuild-model-config/src/sql_values/tests.rs b/crates/sqlbuild-model-config/src/sql_values/tests.rs new file mode 100644 index 0000000000..314ce73d02 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_normalize_sql_value.rs"] +mod normalize_sql_value; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-model-config/src/sql_values/tests/helpers.rs b/crates/sqlbuild-model-config/src/sql_values/tests/helpers.rs new file mode 100644 index 0000000000..f7a7330a0b --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/tests/helpers.rs @@ -0,0 +1,9 @@ +use sqlbuild_core::models::{SqlLogicalType, SqlValueKind}; + +/// The logical type of an integer. +pub(super) fn integer_type() -> SqlLogicalType { + SqlLogicalType { + kind: SqlValueKind::Integer, + element_type: None, + } +} diff --git a/crates/sqlbuild-model-config/src/sql_values/tests/test_normalize_sql_value.rs b/crates/sqlbuild-model-config/src/sql_values/tests/test_normalize_sql_value.rs new file mode 100644 index 0000000000..54bcb46b06 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/tests/test_normalize_sql_value.rs @@ -0,0 +1,67 @@ +use crate::sql_values::errors::SqlValueFailure; +use crate::sql_values::main::normalize_sql_value::normalize_sql_value; +use crate::sql_values::models::{RawSqlValue, SqlValueLimits}; +use crate::sql_values::tests::helpers::integer_type; +use crate::sql_values::tests::test_types::NormalizeTestCase; +use sqlbuild_core::models::SqlValue; + +#[test] +fn given_authored_values_when_normalizing_then_python_canonical_values_and_errors_match() { + let test_cases = [ + NormalizeTestCase { + description: "a set is deduplicated by identity and ordered by its repr", + raw: RawSqlValue::Set(vec![RawSqlValue::Int(2), RawSqlValue::Int(10)]), + explicit_type: None, + expected_value: Ok(SqlValue::Set { + element_type: integer_type(), + values: vec![SqlValue::Integer(10), SqlValue::Integer(2)], + }), + }, + NormalizeTestCase { + description: "a duplicate set value names its payload repr", + raw: RawSqlValue::Set(vec![ + RawSqlValue::Str("GB".to_owned()), + RawSqlValue::Str("GB".to_owned()), + ]), + explicit_type: None, + expected_value: Err(SqlValueFailure::Invalid( + "constant 'x' contains duplicate set value 'GB' at [1]".to_owned(), + )), + }, + NormalizeTestCase { + description: "a quoted decimal keeps its exponent", + raw: RawSqlValue::Str(" 2.4700 ".to_owned()), + explicit_type: Some("decimal"), + expected_value: Ok(SqlValue::Decimal("2.4700".to_owned())), + }, + NormalizeTestCase { + description: "a non-finite float is rejected with its repr", + raw: RawSqlValue::Float(f64::INFINITY), + explicit_type: None, + expected_value: Err(SqlValueFailure::Invalid( + "constant 'x' float must be finite; received inf".to_owned(), + )), + }, + NormalizeTestCase { + description: "a set of decimals beyond Emax overflows like Decimal.normalize", + raw: RawSqlValue::Set(vec![RawSqlValue::Decimal("1E+1000000".to_owned())]), + explicit_type: None, + expected_value: Err(SqlValueFailure::DecimalOverflow), + }, + ]; + + for test_case in test_cases { + let normalized = normalize_sql_value( + &test_case.raw, + "constant 'x'", + test_case.explicit_type, + SqlValueLimits::default(), + ); + + assert_eq!( + normalized, test_case.expected_value, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/sql_values/tests/test_types.rs b/crates/sqlbuild-model-config/src/sql_values/tests/test_types.rs new file mode 100644 index 0000000000..f770038f48 --- /dev/null +++ b/crates/sqlbuild-model-config/src/sql_values/tests/test_types.rs @@ -0,0 +1,10 @@ +use crate::sql_values::errors::SqlValueFailure; +use crate::sql_values::models::RawSqlValue; +use sqlbuild_core::models::SqlValue; + +pub(super) struct NormalizeTestCase { + pub(super) description: &'static str, + pub(super) raw: RawSqlValue, + pub(super) explicit_type: Option<&'static str>, + pub(super) expected_value: Result, +} diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/deferred.rs b/crates/sqlbuild-model-config/src/templates/_helpers/deferred.rs new file mode 100644 index 0000000000..94b4fa3e42 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/_helpers/deferred.rs @@ -0,0 +1,48 @@ +//! The `${CTX:key}` and `${name}` placeholders of a deferred target's schema or database. + +/// The deferred-target context marker inside `${...}`. +const CONTEXT_PREFIX: &str = "CTX:"; +/// The opening of a `${...}` placeholder. +pub(crate) const PLACEHOLDER_OPEN: &str = "${"; +const PLACEHOLDER_CLOSE: char = '}'; + +/// Each `{open}body}` in `text` whose body is non-empty and free of `}` and `forbidden`, in order. +pub(crate) fn placeholders<'t>( + text: &'t str, + open: &str, + forbidden: Option, +) -> Vec<(usize, usize, &'t str)> { + let mut found: Vec<(usize, usize, &str)> = Vec::new(); + let mut from = 0; + while let Some(offset) = text[from..].find(open) { + let start = from + offset; + let body_start = start + open.len(); + let body_length = text[body_start..] + .find(|character: char| character == PLACEHOLDER_CLOSE || Some(character) == forbidden) + .unwrap_or(text.len() - body_start); + let close = body_start + body_length; + if body_length > 0 && text[close..].starts_with(PLACEHOLDER_CLOSE) { + found.push((start, close + 1, &text[body_start..close])); + from = close + 1; + } else { + from = start + 1; + } + } + found +} + +/// `text` with each `${CTX:key}` naming a key in `keys` replaced by `logical`. +pub(crate) fn substitute_context(text: &str, keys: &[String], logical: &str) -> String { + let mut result = String::with_capacity(text.len()); + let mut copied = 0; + let open = format!("{PLACEHOLDER_OPEN}{CONTEXT_PREFIX}"); + for (start, end, key) in placeholders(text, &open, None) { + if keys.iter().any(|known| known == key) { + result.push_str(&text[copied..start]); + result.push_str(logical); + copied = end; + } + } + result.push_str(&text[copied..]); + result +} diff --git a/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs b/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs index 71a3096584..9bf2eb71e9 100644 --- a/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/templates/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod deferred; pub(crate) mod evaluation; pub(crate) mod expressions; pub(crate) mod missing_values; diff --git a/crates/sqlbuild-model-config/src/templates/main/deferred_target_segments.rs b/crates/sqlbuild-model-config/src/templates/main/deferred_target_segments.rs new file mode 100644 index 0000000000..789f6a7488 --- /dev/null +++ b/crates/sqlbuild-model-config/src/templates/main/deferred_target_segments.rs @@ -0,0 +1,25 @@ +//! The text of a deferred target's schema or database template, split around known variables. + +use crate::templates::_helpers::deferred::{PLACEHOLDER_OPEN, placeholders, substitute_context}; +use crate::templates::models::DeferredSegment; + +/// `template` with deferred context keys replaced by `logical`, split at each known `${name}`. +pub fn deferred_target_segments( + template: &str, + logical: &str, + context_keys: &[String], + variables: &[String], +) -> Vec { + let text = substitute_context(template, context_keys, logical); + let mut segments: Vec = Vec::new(); + let mut copied = 0; + for (start, end, name) in placeholders(&text, PLACEHOLDER_OPEN, Some(':')) { + if variables.iter().any(|known| known == name) { + segments.push(DeferredSegment::Text(text[copied..start].to_owned())); + segments.push(DeferredSegment::Variable(name.to_owned())); + copied = end; + } + } + segments.push(DeferredSegment::Text(text[copied..].to_owned())); + segments +} diff --git a/crates/sqlbuild-model-config/src/templates/main/mod.rs b/crates/sqlbuild-model-config/src/templates/main/mod.rs index 2d0b299792..90ea55da0e 100644 --- a/crates/sqlbuild-model-config/src/templates/main/mod.rs +++ b/crates/sqlbuild-model-config/src/templates/main/mod.rs @@ -1,3 +1,4 @@ +pub mod deferred_target_segments; pub mod expand_template_string; pub mod expand_template_text; pub mod template_error_message; diff --git a/crates/sqlbuild-model-config/src/templates/models.rs b/crates/sqlbuild-model-config/src/templates/models.rs index 5b77bd385e..f1d25bea65 100644 --- a/crates/sqlbuild-model-config/src/templates/models.rs +++ b/crates/sqlbuild-model-config/src/templates/models.rs @@ -60,3 +60,10 @@ pub enum StringExpansion { /// The templates inside the string were substituted. Text(String), } + +/// A piece of a deferred target template: literal text, or a variable to render. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DeferredSegment { + Text(String), + Variable(String), +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 7f29826022..52e5fe98d2 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -19,6 +19,7 @@ extension-module = ["pyo3/extension-module"] fensu-policy.workspace = true mimalloc.workspace = true pyo3.workspace = true +serde.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true @@ -27,6 +28,7 @@ sqlbuild-config.workspace = true sqlbuild-core.workspace = true sqlbuild-discovery.workspace = true sqlbuild-model-config.workspace = true +sqlbuild-refactor.workspace = true sqlbuild-render.workspace = true sqlbuild-rules.workspace = true sqlbuild-scopes.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs new file mode 100644 index 0000000000..63c25a40e3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs @@ -0,0 +1,58 @@ +//! A hand-built model's binding diagnostics at their authored locations. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; +use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; +use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; + +/// `(code, message, severity, line, column, (line, column, end line, end column))`. +type LocatedRow = ( + String, + String, + String, + Option, + Option, + Option<(usize, usize, Option, Option)>, +); + +/// Each reported diagnostic of `rows` through `(authored, query, cleaned)` and the expansion. +#[pyfunction] +fn locate_model_binding_diagnostics( + rows: Vec, + texts: (String, String, String), + expansion: Option<(String, Vec>)>, +) -> PyResult> { + let (authored, query, cleaned) = texts; + let source: BindingSource<'_> = BindingSource { + authored: &authored, + query: &query, + cleaned: &cleaned, + expansion: expansion + .as_ref() + .map(|(expanded, passes)| (expanded.as_str(), passes.as_slice())), + }; + let located: Vec = + locate_binding_diagnostics(&rows, &source).map_err(PyValueError::new_err)?; + Ok(located.into_iter().map(located_row).collect()) +} + +fn located_row(item: LocatedDiagnostic) -> LocatedRow { + let span = item + .location + .map(|span| (span.line, span.column, span.end_line, span.end_column)); + ( + item.code, + item.message, + item.severity, + item.line, + item.column, + span, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(locate_model_binding_diagnostics, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index 952f5ba328..20e74c10ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod binding_locations; mod catalog; pub(crate) mod conversions; pub(crate) mod explicit_references; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs index 6a7ccfa676..a8bcdaeb43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -14,6 +14,39 @@ fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult, sql: &str, dialect: Option<&str>) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::sql_tokens::main::compiler_query_fingerprint::compiler_query_fingerprint( + sql, dialect, + ) + }) + .map_err(value_error) +} + +/// Python's `compute_function_definition_hash`. +#[pyfunction] +fn function_definition_hash( + py: Python<'_>, + sql: &str, + language: &str, + dialect: Option<&str>, +) -> PyResult { + py.compiler_detach(|| { + sqlbuild_analysis::sql_tokens::main::definition_fingerprint::definition_fingerprint( + sql, language, dialect, + ) + }) + .map_err(value_error) +} + +/// Python's `normalize_query_sql`: stripped, whitespace runs collapsed. +#[pyfunction] +fn normalize_query_text(sql: &str) -> String { + sqlbuild_analysis::sql_tokens::main::normalized_query_text::normalized_query_text(sql) +} + #[pyfunction(name = "validate_sql_with_schema_json")] fn schema_validation_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { @@ -73,7 +106,11 @@ fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> P pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::analysis::relation_names::register(module)?; crate::bindings::_helpers::analysis::explicit_references::register(module)?; + crate::bindings::_helpers::analysis::binding_locations::register(module)?; module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; + module.add_function(wrap_pyfunction!(compiler_query_hash, module)?)?; + module.add_function(wrap_pyfunction!(function_definition_hash, module)?)?; + module.add_function(wrap_pyfunction!(normalize_query_text, module)?)?; module.add_function(wrap_pyfunction!(schema_validation_json, module)?)?; module.add_function(wrap_pyfunction!(schema_validations_json, module)?)?; module.add_function(wrap_pyfunction!(analyze_sql_uses_json, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs index 179c323596..46f16013cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs @@ -2,16 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use sqlbuild_core::panics::main::native_failure::native_failure; - -use crate::bindings::_helpers::boundary::panics::{compiler_error, value_error}; -use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::types::CompilerDetach; -const HELPER_READS_CONTEXT: &str = "native SQL-test helper reads"; - #[pyfunction] fn render_sql_test_comparisons_json(py: Python<'_>, request_json: &str) -> PyResult { py.compiler_detach(|| { @@ -38,61 +32,7 @@ fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult, - sql: &str, - file_label: &str, - syntax: LexicalSyntaxInput, -) -> PyResult { - let syntax: LexicalSyntax = syntax.into(); - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::sql_scenario_extraction::extract_scenario_json( - sql, file_label, &syntax, - ) - }) - .map_err(value_error) -} - -/// The error for a function type the adapter's SQL analysis dialect cannot parse, or `None`. -#[pyfunction] -fn function_type_error( - py: Python<'_>, - type_sql: &str, - adapter_name: &str, - context: &str, -) -> PyResult> { - py.compiler_detach(|| { - sqlbuild_analysis::compiler::main::function_type_validation::function_type_error( - type_sql, - adapter_name, - context, - ) - }) - .map_err(value_error) -} - -/// Python's `_parsed_reads`: the helper keys a CTE body reads, or None if it does not parse. -#[pyfunction] -fn sql_test_helper_reads( - py: Python<'_>, - sql: &str, - keys: Vec, -) -> PyResult>> { - py.compiler_detach(|| { - sqlbuild_analysis::assembly::analysis_session::main::sql_test_helper_reads::sql_test_helper_reads( - sql, &keys, - ) - .map_err(|reason| native_failure(HELPER_READS_CONTEXT, &reason)) - }) - .map_err(compiler_error) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(sql_test_helper_reads, module)?)?; - module.add_function(wrap_pyfunction!(function_type_error, module)?)?; - module.add_function(wrap_pyfunction!(extract_sql_scenario_json, module)?)?; module.add_function(wrap_pyfunction!(render_sql_test_comparisons_json, module)?)?; module.add_function(wrap_pyfunction!( render_sql_test_difference_sample_json, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs index e631bc54ea..5d597675aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs @@ -1,7 +1,8 @@ -//! Test-only hook comparing native CTE fact recovery with Python's, function for function. +//! Test-only hooks comparing native CTE fact recovery and set-operation search with Python's. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::analysis_session::main::names_set_operation::names_set_operation; use sqlbuild_analysis::assembly::analysis_session::main::recover_cte_facts::recover_cte_facts; use sqlbuild_analysis::assembly::analysis_session::models::{CteFactRequest, CteFacts}; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; @@ -56,7 +57,14 @@ fn _oracle_cte_fact_recovery(py: Python<'_>, request: RecoveryRow) -> PyResult bool { + names_set_operation(sql) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(_oracle_cte_fact_recovery, module)?)?; + module.add_function(wrap_pyfunction!(_oracle_names_set_operation, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs index abd40f0195..2bd572c62a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs @@ -89,7 +89,7 @@ fn infer_query_columns( } /// A normalization rejection as Python's `ValueError`; anything else as an internal failure. -fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { +pub(crate) fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { match failure { QueryColumnsFailure::Normalization(message) => compiler_error(message), QueryColumnsFailure::Internal(reason) => { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 65a1324edc..2277fcdde0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -7,6 +7,8 @@ use sqlbuild_analysis::assembly::analysis_session::main::attach_analysis_cache:: use sqlbuild_analysis::assembly::analysis_session::main::expression_shapes::expression_shapes; use sqlbuild_analysis::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; use sqlbuild_analysis::assembly::analysis_session::main::finished_fact_models::finished_fact_models; +use sqlbuild_analysis::assembly::analysis_session::main::model_reference_facts::model_reference_facts; +use sqlbuild_analysis::assembly::analysis_session::main::names_set_operation::names_set_operation; use sqlbuild_analysis::assembly::analysis_session::main::prove_dynamic_contracts::prove_dynamic_contracts; use sqlbuild_analysis::assembly::analysis_session::main::prove_finished_dynamic_contracts::prove_finished_dynamic_contracts; use sqlbuild_analysis::assembly::analysis_session::main::run_analysis_session::run_analysis_session; @@ -15,12 +17,13 @@ use sqlbuild_analysis::assembly::analysis_session::main::start_analysis_session: use sqlbuild_analysis::assembly::analysis_session::main::take_analysis_cache::take_analysis_cache; use sqlbuild_analysis::assembly::analysis_session::models::{ AnalysisCacheStats, AnalysisSession, ColumnFact, DynamicFamily, ExpressionShape, - ExpressionShapeRequest, FinishedSession, LineageFacts, LineageRow, ModelOutcome, - ModelReference, ModelRequest, PivotBatchRequest, PivotModel, PivotOutcome, PivotTables, - SessionOutcome, SessionRequest, SessionStep, + ExpressionShapeRequest, FinishedSession, LineageFacts, LineageRow, ModelOutcome, ModelRequest, + PivotBatchRequest, PivotModel, PivotOutcome, PivotTables, SessionOutcome, SessionRequest, + SessionStep, }; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_analysis::semantic_validation::types::DiagnosticRow; +use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use sqlbuild_cache::store::models::NativeStore; use sqlbuild_core::constants::PANIC_MESSAGE; @@ -48,13 +51,9 @@ type ModelRow = ( String, String, Pairs, - Vec<(String, bool)>, - Vec<(String, String, String)>, - Vec, - bool, + Vec<(String, String)>, bool, Option<(String, String)>, - String, Vec, ); /// The profile settings, relation shapes and models one session analyses. @@ -109,14 +108,16 @@ type PivotRequestRow = ( type ContractRow = (&'static str, Option); /// `(hits, misses, stored, why the store was not read or saved)`. type CacheStatsRow = (usize, usize, usize, Option); -/// `(models, catalog schema additions, analysis-shape names, dynamic pivot proofs)`. -type FinishRow = (Vec, Shapes, Vec, Vec); +/// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. +type FinishRow = (Shapes, Vec, Vec); /// One compile's native model analysis; an internal failure raises `NativeCompilerError`. #[pyclass(module = "sqlbuild._native")] pub(crate) struct NativeModelAnalysisSession { inner: Option, finished: Option, + /// Each requested model's outcome and dynamic pivot proof, in request order, once finished. + outcomes: Vec<(ModelOutcome, PivotOutcome)>, sharing: (usize, usize), cache_path: Option, cache_stats: Option, @@ -135,6 +136,13 @@ impl NativeModelAnalysisSession { self.finished.as_ref() } + /// The finished outcome and dynamic pivot proof of the model requested at `index`. + pub(crate) fn outcome(&self, index: usize) -> PyResult<&(ModelOutcome, PivotOutcome)> { + self.outcomes + .get(index) + .ok_or_else(|| internal_error("the session returned no analysis for a model")) + } + /// The running session, or `NativeCompilerError` once it has failed or finished. fn running(&mut self) -> PyResult { self.inner @@ -191,7 +199,32 @@ impl NativeModelAnalysisSession { .map(|error| error.to_string()); self.cache_stats = Some((stats.hits, stats.misses, stats.stored, saved)); } - Ok(finish_row(outcome)) + let SessionOutcome { + models, + schema_additions, + analysis_names, + dynamic_contracts, + } = outcome; + let succeeded: Vec = models + .iter() + .map(|model| model.analysis.analysis_succeeded) + .collect(); + self.outcomes = models.into_iter().zip(dynamic_contracts).collect(); + Ok((schema_additions, analysis_names, succeeded)) + } + + /// Each requested model's raw binding diagnostic rows, in request order. + fn binding_rows(&self) -> Vec> { + self.outcomes + .iter() + .map(|(outcome, _)| outcome.analysis.binding_diagnostics.clone()) + .collect() + } + + /// The model at `index`'s outcome and proof rows, for hand-built model inputs only. + fn outcome_rows(&self, index: usize) -> PyResult<(OutcomeRow, ContractRow)> { + let (outcome, contract) = self.outcome(index)?; + Ok((outcome_row(outcome.clone()), contract_row(contract.clone()))) } /// The analysis cache's `(hits, misses, stored, failure)`; None without a cache. @@ -256,6 +289,7 @@ fn start_model_analysis_session( Ok(NativeModelAnalysisSession { inner: Some(session), finished: None, + outcomes: Vec::new(), sharing: (0, 0), cache_path, cache_stats, @@ -335,28 +369,25 @@ fn model_request(row: ModelRow) -> ModelRequest { name, query_sql, placeholders, - references, - lineage_references, - required_names, + reference_rows, recover_cte_facts, - has_set_operation, snapshot_columns, - pivot_sql, dynamic_families, ) = row; + let has_set_operation = names_set_operation(&query_sql); + let facts = model_reference_facts(&reference_rows); + let pivot_sql = if placeholders.is_empty() { + query_sql.clone() + } else { + placeholder_defaults(&query_sql, &placeholders) + }; ModelRequest { name, query_sql, placeholders, - references: references - .into_iter() - .map(|(analysis_name, model_ref)| ModelReference { - analysis_name, - model_ref, - }) - .collect(), - lineage_references, - required_names, + references: facts.references, + lineage_references: facts.lineage_references, + required_names: facts.required_names, recover_cte_facts, has_set_operation, snapshot_columns, @@ -386,19 +417,6 @@ fn lineage_items(rows: Vec) -> Vec { .collect() } -fn finish_row(outcome: SessionOutcome) -> FinishRow { - ( - outcome.models.into_iter().map(outcome_row).collect(), - outcome.schema_additions, - outcome.analysis_names, - outcome - .dynamic_contracts - .into_iter() - .map(contract_row) - .collect(), - ) -} - fn families_by_table(rows: Vec<(String, Vec)>) -> Vec<(String, Vec)> { rows.into_iter() .map(|(name, families)| (name, families.into_iter().map(dynamic_family).collect())) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/audit_rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/audit_rendering.rs deleted file mode 100644 index 5d752616fe..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/audit_rendering.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! Attached generic audit rendering for the preview compile attachments. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::PyDict; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_attachments::audits::main::render_attached_audit::render_attached_audit; -use sqlbuild_attachments::audits::models::{AuditAttachment, AuditRendering, PolicySource}; - -use crate::bindings::_helpers::attachments::argument_values::argument_pairs; -use crate::bindings::_helpers::boundary::panics::compiler_guard; - -/// `(render error, sql body, evidence sql, severity, run scope source, policy error)`. -type RenderedRow = ( - Option, - String, - Option, - &'static str, - &'static str, - Option, -); - -/// Authored policies of one attachment: mode and threshold flags, severities and run scopes. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct AuditPolicies { - has_thresholds: bool, - threshold_error: bool, - instance_severity: Option, - default_severity: Option, - instance_run_scope: Option, - default_run_scope: Option, -} - -/// Render one `(owner, audit)` attachment, or the error rendering raises. -#[pyfunction] -fn render_attached_generic_audit( - labels: (String, String), - sql: (String, Option), - arguments: (Vec<(String, String)>, Bound<'_, PyDict>), - policies: AuditPolicies, -) -> PyResult { - compiler_guard(|| { - let (implicit, explicit) = (arguments.0, argument_pairs(&arguments.1)?); - let attachment: AuditAttachment = AuditAttachment { - owner_label: labels.0, - definition_name: labels.1, - sql_body: sql.0, - evidence_sql: sql.1, - implicit_arguments: implicit, - explicit_arguments: explicit, - has_thresholds: policies.has_thresholds, - threshold_error: policies.threshold_error, - instance_severity: policies.instance_severity, - default_severity: policies.default_severity, - instance_run_scope: policies.instance_run_scope, - default_run_scope: policies.default_run_scope, - }; - Ok(rendered_row(render_attached_audit(&attachment))) - }) -} - -fn rendered_row(rendering: AuditRendering) -> RenderedRow { - let rendered = match rendering { - AuditRendering::Failed(message) => { - return (Some(message), String::new(), None, "", "", None); - } - AuditRendering::Rendered(rendered) => rendered, - }; - let (severity, source, policy_error): (&'static str, &'static str, Option) = - match rendered.policies { - Ok((severity, source)) => (severity, source_name(source), None), - Err(message) => ("", "", Some(message)), - }; - ( - None, - rendered.sql_body, - rendered.evidence_sql, - severity, - source, - policy_error, - ) -} - -fn source_name(source: PolicySource) -> &'static str { - match source { - PolicySource::Instance => "instance", - PolicySource::Default => "default", - PolicySource::Fallback => "fallback", - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(render_attached_generic_audit, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs index 27f029da1f..94984bb776 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs @@ -3,10 +3,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::types::{PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; -use sqlbuild_attachments::cursor_intrinsics::models::{IntrinsicCheck, IntrinsicModel}; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use sqlbuild_core::text::models::PythonText; use crate::bindings::_helpers::boundary::panics::compiler_guard; @@ -22,25 +21,6 @@ fn semantics((version, unicode): PythonRelease) -> PyResult { python_semantics(version, &unicode) } -/// `None` if `sql` is free of cursor intrinsics and reserved markers, else the error for `context`. -#[pyfunction] -fn cursor_intrinsics_rejection( - sql: &str, - reserved_markers: Vec, - context: &str, - python: PythonRelease, -) -> PyResult> { - let python: PythonText = semantics(python)?; - compiler_guard(|| { - Ok( - match intrinsic_free(python, sql, &reserved_markers, context) { - IntrinsicCheck::Free => None, - IntrinsicCheck::Rejected(message) => Some(message), - }, - ) - }) -} - /// A model query with canonical intrinsic calls and no error, or no query and the error. #[pyfunction] fn validated_model_cursor_intrinsics( @@ -104,7 +84,6 @@ fn replace_cursor_intrinsics( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(cursor_intrinsics_rejection, module)?)?; module.add_function(wrap_pyfunction!(validated_model_cursor_intrinsics, module)?)?; module.add_function(wrap_pyfunction!(replace_cursor_intrinsics, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/function_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/function_headers.rs index 9e8845c0f8..98c8087ea6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/function_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/function_headers.rs @@ -1,110 +1,30 @@ -//! SQL and Python function header parsing and namespaces for the preview compile attachments. +//! Function header values as the native function attachment parses them. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods}; use pyo3::types::{ PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods, }; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_attachments::functions::main::parse_function_header::parse_function_header; -use sqlbuild_attachments::functions::main::resolve_function_namespace::resolve_function_namespace; -use sqlbuild_attachments::functions::models::{ - FunctionHeader, FunctionLanguage, FunctionNamespace, FunctionReturns, HeaderStage, HeaderValue, - NamedType, NamespaceInputs, -}; - -use crate::bindings::_helpers::boundary::panics::compiler_guard; - -/// `(authored name, name, type)` rows. -type Pairs = Vec<(String, String, String)>; -/// Parsed header fields in Python's order, then its first error as `(stage, message)`. -type HeaderRow = ( - Pairs, - Option, - Option, - Vec, - Option, - Option, - Option, - Vec, - Option<(&'static str, String)>, -); -type NamespaceRow = [Option; 8]; - -/// The expanded namespace inputs of one function. -#[derive(FromPyObject)] -#[pyo3(from_item_all)] -struct NamespaceInput { - header_database: Option, - header_schema: Option, - default_database: Option, - default_schema: Option, - target_database: Option, - target_schema: Option, - python: bool, - inherit_default_namespace: bool, -} +use sqlbuild_attachments::functions::models::{FunctionHeader, FunctionLanguage, HeaderValue}; -/// Parse one function file's header with its first error; non-string keys are never read. -#[pyfunction] -fn parse_function_header_values( - header_values: Bound<'_, PyDict>, - python: bool, +/// One function file's parsed header; non-string keys and `None` values are never read. +pub(crate) fn parsed_header( + header_values: &Bound<'_, PyDict>, + language: FunctionLanguage, relative_path: &str, -) -> PyResult { - compiler_guard(|| { - let mut header: Vec<(String, HeaderValue)> = Vec::with_capacity(header_values.len()); - for (key, value) in header_values.iter() { - let Some(name) = text_value(&key) else { - continue; - }; - if value.is_none() { - continue; - } - header.push((name, header_value(&value))); - } - let language: FunctionLanguage = if python { - FunctionLanguage::Python - } else { - FunctionLanguage::Sql +) -> FunctionHeader { + let mut header: Vec<(String, HeaderValue)> = Vec::with_capacity(header_values.len()); + for (key, value) in header_values.iter() { + let Some(name) = text_value(&key) else { + continue; }; - Ok(header_row(parse_function_header( - &header, - language, - relative_path, - ))) - }) -} - -/// Python's physical, logical and fingerprint database and schema of one function. -#[pyfunction] -fn resolve_function_namespace_values(inputs: NamespaceInput) -> PyResult { - compiler_guard(|| { - let namespace: FunctionNamespace = resolve_function_namespace(&NamespaceInputs { - header_database: inputs.header_database, - header_schema: inputs.header_schema, - default_database: inputs.default_database, - default_schema: inputs.default_schema, - target_database: inputs.target_database, - target_schema: inputs.target_schema, - language: if inputs.python { - FunctionLanguage::Python - } else { - FunctionLanguage::Sql - }, - inherit_default_namespace: inputs.inherit_default_namespace, - }); - Ok([ - namespace.database, - namespace.schema, - namespace.logical_database, - namespace.logical_schema, - namespace.fingerprint_database, - namespace.fingerprint_schema, - namespace.fingerprint_logical_database, - namespace.fingerprint_logical_schema, - ]) - }) + if value.is_none() { + continue; + } + header.push((name, header_value(&value))); + } + parse_function_header(&header, language, relative_path) } /// One header value. @@ -136,48 +56,3 @@ fn text_value(value: &Bound<'_, PyAny>) -> Option { Err(_) => None, } } - -fn header_row(header: FunctionHeader) -> HeaderRow { - let (scalar, table): (Option, Option) = match header.returns { - Some(FunctionReturns::Type(text)) => (Some(text), None), - Some(FunctionReturns::Table(columns)) => (None, Some(rows(columns))), - None => (None, None), - }; - ( - rows(header.arguments), - scalar, - table, - header.tags, - header.description, - header.runtime_version, - header.entry_point, - header.packages, - header - .failure - .map(|failure| (stage_name(failure.stage), failure.message)), - ) -} - -fn stage_name(stage: HeaderStage) -> &'static str { - match stage { - HeaderStage::Start => "start", - HeaderStage::Arguments => "arguments", - HeaderStage::Returns => "returns", - HeaderStage::PythonValues => "python_values", - HeaderStage::Metadata => "metadata", - } -} - -fn rows(named: Vec) -> Pairs { - let mut rows: Pairs = Vec::with_capacity(named.len()); - for item in named { - rows.push((item.raw_name, item.name, item.type_text)); - } - rows -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_function_header_values, module)?)?; - module.add_function(wrap_pyfunction!(resolve_function_namespace_values, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs index 8b62154217..06178594d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs @@ -1,9 +1,6 @@ pub(crate) mod argument_values; -pub(crate) mod audit_rendering; pub(crate) mod ceremonial_select; pub(crate) mod cursor_intrinsics; pub(crate) mod function_headers; +pub(crate) mod reference_markers; pub(crate) mod registration; -pub(crate) mod seed_pairing; -pub(crate) mod test_parameters; -pub(crate) mod test_targets; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs new file mode 100644 index 0000000000..d22fdd8dd0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs @@ -0,0 +1,29 @@ +//! The first reference call executable SQL still contains. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyString, PyStringMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_attachments::reference_markers::main::unresolved_reference_marker::unresolved_reference_marker; +use sqlbuild_attachments::sql_lexing::main::unclosed_message::unclosed_message; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +/// The kind index of the first unresolved reference call, or the unclosed-construct message. +#[pyfunction] +fn unresolved_reference_marker_kind( + sql: &Bound<'_, PyString>, + context: &str, +) -> PyResult<(Option, Option)> { + let text = sql.to_string_lossy(); + compiler_guard(|| { + Ok(match unresolved_reference_marker(&text) { + Ok(marker) => (marker, None), + Err(construct) => (None, Some(unclosed_message(context, construct))), + }) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(unresolved_reference_marker_kind, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs index 78cc73e983..2f62e267e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs @@ -3,17 +3,12 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::attachments::{ - audit_rendering, ceremonial_select, cursor_intrinsics, function_headers, seed_pairing, - test_parameters, test_targets, + ceremonial_select, cursor_intrinsics, reference_markers, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - audit_rendering::register(module)?; - seed_pairing::register(module)?; cursor_intrinsics::register(module)?; - test_parameters::register(module)?; ceremonial_select::register(module)?; - test_targets::register(module)?; - function_headers::register(module)?; + reference_markers::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/seed_pairing.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/seed_pairing.rs deleted file mode 100644 index f9dcbb3fe6..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/seed_pairing.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Seed declaration pairing for the preview compile attachments. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; - -use crate::bindings::_helpers::boundary::panics::compiler_guard; - -/// Pair declarations with seed file stems: `(file indices, None)` or `(None, missing index)`. -#[pyfunction] -fn pair_seed_files( - declarations: Vec, - stems: Vec, -) -> PyResult<(Option>, Option)> { - compiler_guard(|| { - Ok(match pair_seed_declarations(&declarations, &stems) { - Ok(pairs) => (Some(pairs), None), - Err(missing) => (None, Some(missing)), - }) - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(pair_seed_files, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_parameters.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_parameters.rs deleted file mode 100644 index 2d344fc816..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_parameters.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! SQL test `@param("name")` reference scanning for the preview compile attachments. - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyfunction, wrap_pyfunction}; -use sqlbuild_attachments::test_parameters::main::parameter_references::parameter_references; - -use crate::bindings::_helpers::boundary::panics::compiler_guard; - -/// `(start, end, name)` by code points. -type Span = (usize, usize, String); - -/// The references Python renders and the error it then raises for `owner`, the test case. -#[pyfunction] -fn scan_test_parameter_references( - sql: &str, - declared: Vec, - owner: &str, -) -> PyResult<(Vec, Option)> { - compiler_guard(|| { - let scan = parameter_references(sql, &declared, owner); - let mut spans: Vec = Vec::with_capacity(scan.references.len()); - for reference in scan.references { - spans.push((reference.start, reference.end, reference.name)); - } - Ok((spans, scan.error)) - }) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scan_test_parameter_references, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_targets.rs deleted file mode 100644 index 822a394702..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_targets.rs +++ /dev/null @@ -1,110 +0,0 @@ -//! SQL test target and scenario source validation for the preview compile attachments. - -use std::collections::HashSet; - -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; -use pyo3::{pyclass, pymethods}; -use sqlbuild_attachments::test_targets::main::scenario_source_violation::scenario_source_violation; -use sqlbuild_attachments::test_targets::main::unknown_test_target::unknown_test_target; -use sqlbuild_attachments::test_targets::models::{ScenarioCteSources, TargetCatalog, TestTargets}; - -use crate::bindings::_helpers::boundary::panics::compiler_guard; - -/// A model test's target name groups, in `TestTargets` field order. -type TargetRow = ( - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, - Vec, -); - -/// The project's resource names, read once per compile and shared by every test and scenario. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct SqlTestTargetCatalog { - catalog: TargetCatalog, -} - -#[pymethods] -impl SqlTestTargetCatalog { - #[new] - fn new( - models: HashSet, - sources: HashSet, - seeds: HashSet, - resources: (HashSet, HashSet), - ) -> Self { - let (table_functions, macros) = resources; - Self { - catalog: TargetCatalog { - models, - sources, - seeds, - table_functions, - macros, - }, - } - } - - /// Python's error for a model test's first unknown target, or `None`. - fn unknown_test_target( - &self, - file_label: &str, - targets: TargetRow, - ) -> PyResult> { - compiler_guard(|| { - let ( - mock_models, - mock_sources, - mock_seeds, - mock_table_functions, - macro_mocks, - expected_models, - assertion_targets, - ) = targets; - Ok(unknown_test_target( - file_label, - &self.catalog, - &TestTargets { - mock_models, - mock_sources, - mock_seeds, - mock_table_functions, - macro_mocks, - expected_models, - assertion_targets, - }, - )) - }) - } - - /// Python's error for a scenario's first disallowed `(cte, is check, sources)` read, or `None`. - fn scenario_source_error( - &self, - file_label: &str, - ctes: Vec<(String, bool, Vec)>, - ) -> PyResult> { - compiler_guard(|| { - let mut cte_sources: Vec = Vec::with_capacity(ctes.len()); - for (name, check, sources) in ctes { - cte_sources.push(ScenarioCteSources { - name, - check, - sources, - }); - } - Ok(scenario_source_violation( - file_label, - &cte_sources, - &self.catalog.sources, - )) - }) - } -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/oracles.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/oracles.rs index 00e219eafa..c3cd64d66d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/oracles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/oracles.rs @@ -6,8 +6,6 @@ use pyo3::{PyErr, pyfunction, wrap_pyfunction}; use serde_json::{Value, json}; use sqlbuild_config::errors::ConfigError; use sqlbuild_config::models::{ConfigDate, ConfigTime, ConfigValue}; -use sqlbuild_config::project::main::read_local_config::read_local_config; -use sqlbuild_config::project::main::read_project_config::read_project_config; use sqlbuild_config::toml::main::load_toml::load_toml; use sqlbuild_config::yaml::main::safe_load::safe_load; use sqlbuild_core::json::main::dumps::dumps; @@ -20,7 +18,6 @@ use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; use sqlbuild_core::text::main::python_text::python_text; use sqlbuild_core::text::models::LineIndex; -use std::path::Path; fn oracle_error(message: impl std::fmt::Display) -> PyErr { PyValueError::new_err(message.to_string()) @@ -204,32 +201,6 @@ fn _oracle_toml_load(text: &str) -> String { outcome(load_toml(text).map(|value| canonical(&value))) } -/// Read a project's discovery configuration natively, or return `{"error": kind}`. -#[pyfunction] -fn _oracle_project_config(project_dir: &str) -> String { - let directory = Path::new(project_dir); - outcome(read_project_config(directory).and_then(|project| { - let local = read_local_config(directory)?; - Ok(json!({ - "name": project.name, - "adapter": project.adapter, - "default_target": project.default_target, - "sql_analysis": project.settings.sql_analysis, - "require_sql_analysis": project.settings.require_sql_analysis, - "enforce_placement": project.enforce_placement, - "enforce_explicit_references": project.enforce_explicit_references, - "vars": project.vars, - "path_default_keys": project.path_defaults.iter().map(|(key, _)| key).collect::>(), - "target_names": project.target_names, - "local_target": local.target, - "local_adapter": local.adapter, - "local_sql_analysis": local.sql_analysis, - "local_vars": local.vars, - "local_target_names": local.target_names, - })) - })) -} - #[pyfunction] fn _oracle_python_alnum( python_version: (u8, u8), @@ -282,6 +253,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(_oracle_text_positions, module)?)?; module.add_function(wrap_pyfunction!(_oracle_yaml_load, module)?)?; module.add_function(wrap_pyfunction!(_oracle_toml_load, module)?)?; - module.add_function(wrap_pyfunction!(_oracle_project_config, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs index fd2e81a45c..090b854fed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs @@ -3,10 +3,12 @@ use pyo3::exceptions::{PyOSError, PyRuntimeError}; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cache::digest::main::compiled_code_identity::compiled_code_identity as compiled; use sqlbuild_cache::digest::main::content_digest::content_digest as digest; use sqlbuild_cache::digest::main::digest_files::digest_files as digest_each; use sqlbuild_cache::digest::main::fingerprint_project_files::fingerprint_project_files as fingerprint; use sqlbuild_cache::digest::main::hex_digest::hex_digest; +use sqlbuild_cache::digest::main::installed_code_identity::installed_code_identity as installed; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::main::open_native_store::open_native_store; use sqlbuild_cache::store::models::NativeStore; @@ -80,7 +82,26 @@ pub(crate) fn save_store( .map_err(|error| PyOSError::new_err(error.to_string())) } +/// Digest of the installed code's facts and, for an editable install, its source files. +#[pyfunction] +pub(crate) fn installed_code_identity( + py: Python<'_>, + facts: Vec, + package_root: Option, +) -> PyResult { + py.compiler_detach(|| Ok(installed(&facts, package_root.as_deref()))) + .map_err(PyRuntimeError::new_err) +} + +/// Digest of the installed code's identity and the native build's identity. +#[pyfunction] +pub(crate) fn compiled_code_identity(installed: &str, build_identity: &str) -> String { + compiled(installed, build_identity) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(installed_code_identity, module)?)?; + module.add_function(wrap_pyfunction!(compiled_code_identity, module)?)?; module.add_function(wrap_pyfunction!(content_digest, module)?)?; module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?; module.add_function(wrap_pyfunction!(digest_files, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs index d53b762eea..6c69245ecd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs @@ -4,21 +4,30 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyclass, pymethods}; use sqlbuild_cache::digest::main::content_digest::content_digest; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_cache::digest::types::ContentDigest; use sqlbuild_cache::store::models::NativeStore; use std::path::PathBuf; use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; /// The native store kind holding SQL-test scan results. const SQL_TEST_SCAN_STORE_KIND: &str = "sql-test-scans"; +/// The store file below the compile cache folder. +const SQL_TEST_SCAN_STORE_FILE_NAME: &str = "sql-test-scans.bin"; +/// Bumped whenever stored scan values change shape; older stores open empty. +const SQL_TEST_SCAN_STORE_VERSION: &str = "sql-test-scan-store-v1"; /// One compile's view of the stored scan results, keyed by the digest of the caller's key parts. #[pyclass(module = "sqlbuild._native", frozen)] #[derive(Debug)] pub(crate) struct SqlTestScanStore { store: Mutex, + path: PathBuf, + hits: AtomicUsize, + misses: AtomicUsize, } fn poisoned(_: T) -> pyo3::PyErr { @@ -27,41 +36,86 @@ fn poisoned(_: T) -> pyo3::PyErr { #[pymethods] impl SqlTestScanStore { - /// Open the store at `path` for `environment`; missing, damaged or foreign files open empty. - #[new] - fn new(py: Python<'_>, path: PathBuf, environment: &str) -> PyResult { - let store: NativeStore = open_store(py, &path, SQL_TEST_SCAN_STORE_KIND, environment)?; - Ok(Self { - store: Mutex::new(store), + /// Open the compile cache folder's store for `code_identity`; the error when it cannot open. + #[staticmethod] + fn open_in( + py: Python<'_>, + cache_dir: PathBuf, + code_identity: &str, + ) -> (Option, Option) { + let path: PathBuf = cache_dir.join(SQL_TEST_SCAN_STORE_FILE_NAME); + let environment: String = hex_digest(&content_digest(&[ + SQL_TEST_SCAN_STORE_VERSION, + code_identity, + ])); + match open_store(py, &path, SQL_TEST_SCAN_STORE_KIND, &environment) { + Ok(store) => ( + Some(Self { + store: Mutex::new(store), + path, + hits: AtomicUsize::new(0), + misses: AtomicUsize::new(0), + }), + None, + ), + Err(error) => (None, Some(error.to_string())), + } + } + + /// The hits and misses counted by this compile. + fn counts(&self) -> (usize, usize) { + ( + self.hits.load(Ordering::Relaxed), + self.misses.load(Ordering::Relaxed), + ) + } + + /// Atomically save values this compile stored; entries written, or the error that stopped it. + fn save(&self, py: Python<'_>) -> PyResult<(Option, Option)> { + let store = self.store.lock().map_err(poisoned)?; + Ok(match save_store(py, &store, &self.path, &[]) { + Ok(written) => (written, None), + Err(error) => (None, Some(error.to_string())), }) } +} - /// The value stored under the digest of `key_parts`, or `None`. - fn get(&self, key_parts: Vec) -> PyResult>> { +impl SqlTestScanStore { + /// The stored value under `key_parts`, counted as a hit when `decode` accepts it. + pub(crate) fn decoded( + &self, + key_parts: Vec, + decode: impl FnOnce(&[u8]) -> Option, + ) -> PyResult> { let key: ContentDigest = content_digest(&key_parts); - Ok(self + let decoded: Option = self .store .lock() .map_err(poisoned)? .get(&key) - .map(<[u8]>::to_vec)) + .and_then(decode); + let counter: &AtomicUsize = if decoded.is_some() { + &self.hits + } else { + &self.misses + }; + counter.fetch_add(1, Ordering::Relaxed); + Ok(decoded) } - /// Store `value` under the digest of `key_parts`. - fn put(&self, key_parts: Vec, value: Vec) -> PyResult<()> { + /// Store one complete result under `key_parts`. + pub(crate) fn stored(&self, key_parts: Vec, value: Vec) -> PyResult<()> { let key: ContentDigest = content_digest(&key_parts); self.store.lock().map_err(poisoned)?.put(key, value); Ok(()) } - - /// Atomically save the store when this compile stored a value; returns entries written. - fn save(&self, py: Python<'_>, path: PathBuf) -> PyResult> { - let store = self.store.lock().map_err(poisoned)?; - save_store(py, &store, &path, &[]) - } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; + module.add( + "SQL_TEST_SCAN_STORE_FILE_NAME", + SQL_TEST_SCAN_STORE_FILE_NAME, + )?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs new file mode 100644 index 0000000000..228e421fdf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs @@ -0,0 +1,139 @@ +//! Write, clean and publish compiled artifacts natively for the Python target writer. + +use std::collections::HashSet; +use std::path::PathBuf; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyBytes; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cache::artifacts::errors::ArtifactError; +use sqlbuild_cache::artifacts::main::publish_staged::publish_staged; +use sqlbuild_cache::artifacts::main::remove_stale::remove_stale; +use sqlbuild_cache::artifacts::main::staged_files::staged_files; +use sqlbuild_cache::artifacts::main::write_artifacts::write_artifacts; +use sqlbuild_cache::artifacts::models::{Publication, WrittenArtifacts}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +type PublishedRow = (String, Vec<(String, String)>); + +/// Write each `(path, contents)` artifact; returns how many were written and left unchanged. +#[pyfunction] +fn write_compiled_artifacts( + py: Python<'_>, + files: Vec<(PathBuf, Vec)>, + check_existing: bool, +) -> PyResult<(usize, usize)> { + compiler_guard(|| { + let outcome: Result = + py.detach(|| write_artifacts(&files, check_existing)); + let written: WrittenArtifacts = outcome.map_err(|error| artifact_error(py, error))?; + Ok((written.written, written.unchanged)) + }) +} + +/// Delete unmanaged compiled files and empty directories; returns how many files were removed. +#[pyfunction] +fn remove_stale_artifacts( + py: Python<'_>, + compiled_dir: PathBuf, + managed: Vec, +) -> PyResult { + compiler_guard(|| { + let managed: HashSet = managed.into_iter().collect(); + py.detach(|| remove_stale(&compiled_dir, &managed)) + .map_err(|error| artifact_error(py, error)) + }) +} + +/// Publish staged artifacts: `("changed", [])`, `("tree", [])` or `("files", [(source, path)])`. +#[pyfunction] +fn publish_staged_artifacts( + py: Python<'_>, + staged_dir: PathBuf, + compiled_dir: PathBuf, + expected: Vec, +) -> PyResult { + compiler_guard(|| { + let expected: HashSet = expected.into_iter().collect(); + let publication: Publication = py + .detach(|| publish_staged(&staged_dir, &compiled_dir, &expected)) + .map_err(|error| artifact_error(py, error))?; + Ok(match publication { + Publication::StagedChanged => ("changed".to_owned(), Vec::new()), + Publication::MovedTree => ("tree".to_owned(), Vec::new()), + Publication::Files(files) => ( + "files".to_owned(), + files + .into_iter() + .map(|(source, path)| (lossy(&source), lossy(&path))) + .collect(), + ), + }) + }) +} + +/// Every staged artifact's path relative to `staged_dir`. +#[pyfunction] +fn staged_artifact_files(py: Python<'_>, staged_dir: PathBuf) -> PyResult> { + compiler_guard(|| { + let files: Vec = py.detach(|| staged_files(&staged_dir)); + Ok(files.iter().map(|path| lossy(path)).collect()) + }) +} + +fn lossy(path: &std::path::Path) -> String { + path.to_string_lossy().into_owned() +} + +/// The exception Python's writer raises for the same failure. +fn artifact_error(py: Python<'_>, error: ArtifactError) -> PyErr { + match error { + ArtifactError::Io { path, error } => match error.raw_os_error() { + Some(code) => PyOSError::new_err((code, strerror(py, code), lossy(&path))), + None => PyOSError::new_err(error.to_string()), + }, + ArtifactError::Move { + source, + path, + error, + } => match error.raw_os_error() { + Some(code) => PyOSError::new_err(( + code, + strerror(py, code), + lossy(&source), + py.None(), + lossy(&path), + )), + None => PyOSError::new_err(error.to_string()), + }, + ArtifactError::ExistingNotUtf8 { path } => existing_decode_error(py, &path), + } +} + +fn strerror(py: Python<'_>, code: i32) -> String { + py.import("os") + .and_then(|os| os.call_method1("strerror", (code,))) + .and_then(|text| text.extract::()) + .unwrap_or_default() +} + +/// Decode the existing file in Python so the error is exactly the one `bytes.decode` raises. +fn existing_decode_error(py: Python<'_>, path: &PathBuf) -> PyErr { + let bytes: Vec = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(error) => return PyOSError::new_err(error.to_string()), + }; + match PyBytes::new(py, &bytes).call_method1("decode", ("utf-8",)) { + Err(error) => error, + Ok(_) => PyOSError::new_err(format!("compiled artifact '{}' changed", path.display())), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(staged_artifact_files, module)?)?; + module.add_function(wrap_pyfunction!(write_compiled_artifacts, module)?)?; + module.add_function(wrap_pyfunction!(remove_stale_artifacts, module)?)?; + module.add_function(wrap_pyfunction!(publish_staged_artifacts, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs new file mode 100644 index 0000000000..d96a085d1f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs @@ -0,0 +1,242 @@ +//! Emit a command's JSON report as orjson would, or as `json.dumps` does when orjson rejects it. + +use pyo3::exceptions::{PyRecursionError, PyTypeError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, + PyString, PyStringMethods, PyTuple, PyTypeMethods, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{ + JsonDialect, JsonInteger, JsonValue, OrjsonOptions, StdlibJsonOptions, +}; + +use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_guard}; + +const ORJSON_SURROGATE: &str = "str is not valid UTF-8: surrogates not allowed"; +const ORJSON_WIDE_INTEGER: &str = "Integer exceeds 64-bit range"; +const ORJSON_KEY: &str = "Dict key must be str"; +const ORJSON_RECURSION: &str = "Recursion limit reached"; +/// orjson refuses a dict, or a non-empty list or tuple, nested this many containers deep. +const ORJSON_MAX_CONTAINER_DEPTH: usize = 254; +/// CPython's default `sys.getrecursionlimit()`; `json.dumps(indent=2)` recurses once per level. +const PYTHON_RECURSION_LIMIT: usize = 1_000; +const PYTHON_RECURSION_MESSAGE: &str = "maximum recursion depth exceeded"; +const SURROGATE_KEY: &str = "a JSON report key holds a lone surrogate, which report keys never do"; +const ENCODE: &str = "encode"; +const UTF16_LE: &str = "utf-16-le"; +const SURROGATE_PASS: &str = "surrogatepass"; +const INDENTED: OrjsonOptions = OrjsonOptions { + indent_2: true, + sort_keys: false, +}; + +/// What converting the report found besides its values. +#[derive(Default)] +struct Conversion { + /// The `TypeError` orjson raises at the first value it rejects. + orjson_error: Option, + /// The error `json.dumps` raises at the first value it cannot encode either. + stdlib_error: Option, + /// Containers entered above the value being converted. + depth: usize, +} + +/// The report as `orjson.dumps(report, option=OPT_INDENT_2)`, raising orjson's `TypeError`. +#[pyfunction] +fn emit_orjson_report(report: &Bound<'_, PyAny>) -> PyResult { + compiler_guard(|| { + let mut conversion: Conversion = Conversion::default(); + let value: JsonValue = conversion.json_value(report)?; + if let Some(error) = conversion.orjson_error { + return Err(PyTypeError::new_err(error)); + } + emitted(&value, &JsonDialect::Orjson(INDENTED)) + }) +} + +/// The report as orjson writes it, or as `json.dumps(indent=2)` does when orjson rejects it. +#[pyfunction] +fn emit_json_report(report: &Bound<'_, PyAny>) -> PyResult { + compiler_guard(|| { + let mut conversion: Conversion = Conversion::default(); + let value: JsonValue = conversion.json_value(report)?; + if conversion.orjson_error.is_none() { + return emitted(&value, &JsonDialect::Orjson(INDENTED)); + } + if let Some(error) = conversion.stdlib_error { + return Err(error); + } + emitted( + &value, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + ) + }) +} + +fn emitted(value: &JsonValue, dialect: &JsonDialect) -> PyResult { + dumps(value, dialect).map_err(|error| PyTypeError::new_err(format!("{error:?}"))) +} + +impl Conversion { + fn orjson_rejects(&mut self, error: String) { + self.orjson_error.get_or_insert(error); + } + + fn stdlib_rejects(&mut self, error: PyErr) { + self.stdlib_error.get_or_insert(error); + } + + /// Enter a container under both recursion limits; `false` means do not descend into it. + fn enter(&mut self, counted_by_orjson: bool) -> bool { + if counted_by_orjson && self.depth >= ORJSON_MAX_CONTAINER_DEPTH { + self.orjson_rejects(ORJSON_RECURSION.to_owned()); + } + if self.depth >= PYTHON_RECURSION_LIMIT { + self.stdlib_rejects(PyRecursionError::new_err(PYTHON_RECURSION_MESSAGE)); + return false; + } + self.depth += 1; + true + } + + fn json_value(&mut self, value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return self.integer(value); + } + if value.is_instance_of::() { + return Ok(JsonValue::Float(value.extract::()?)); + } + if let Ok(text) = value.downcast::() { + return self.text(text); + } + if value.is_instance_of::() || value.is_instance_of::() { + return self.array(value); + } + if let Ok(mapping) = value.downcast::() { + return self.object(mapping); + } + let name: String = value.get_type().name()?.to_string(); + self.orjson_rejects(format!("Type is not JSON serializable: {name}")); + self.stdlib_rejects(PyTypeError::new_err(format!( + "Object of type {name} is not JSON serializable" + ))); + Ok(JsonValue::Null) + } + + /// The string; one with a lone surrogate, which orjson rejects, as its UTF-16 code units. + fn text(&mut self, text: &Bound<'_, PyString>) -> PyResult { + if let Ok(text) = text.to_str() { + return Ok(JsonValue::String(text.to_owned())); + } + self.orjson_rejects(ORJSON_SURROGATE.to_owned()); + let encoded: Bound<'_, PyBytes> = text + .call_method1(ENCODE, (UTF16_LE, SURROGATE_PASS))? + .downcast_into::()?; + Ok(JsonValue::Utf16( + encoded + .as_bytes() + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) + .collect(), + )) + } + + /// orjson encodes 64-bit integers; wider ones make it raise, so `json.dumps` writes them. + fn integer(&mut self, value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(signed) = value.extract::() { + return Ok(JsonValue::Integer(JsonInteger::from(signed))); + } + if let Ok(unsigned) = value.extract::() { + return Ok(JsonValue::Integer(JsonInteger::from(unsigned))); + } + self.orjson_rejects(ORJSON_WIDE_INTEGER.to_owned()); + let decimal: String = value.str()?.to_str()?.to_owned(); + Ok(JsonInteger::parse(&decimal).map_or(JsonValue::Null, JsonValue::Integer)) + } + + fn array(&mut self, value: &Bound<'_, PyAny>) -> PyResult { + if !self.enter(value.len()? > 0) { + return Ok(JsonValue::Null); + } + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(self.json_value(&item?)?); + } + self.depth -= 1; + Ok(JsonValue::Array(items)) + } + + fn object(&mut self, mapping: &Bound<'_, PyDict>) -> PyResult { + if !self.enter(true) { + return Ok(JsonValue::Null); + } + let mut entries: Vec<(String, JsonValue)> = Vec::with_capacity(mapping.len()); + for (key, item) in mapping.iter() { + let key: String = self.object_key(&key)?; + let item: JsonValue = self.json_value(&item)?; + entries.push((key, item)); + } + self.depth -= 1; + Ok(JsonValue::Object(entries)) + } + + /// A string key, or the text `json.dumps` writes for a scalar key orjson rejects. + fn object_key(&mut self, key: &Bound<'_, PyAny>) -> PyResult { + if let Ok(text) = key.downcast::() { + return match self.text(text)? { + JsonValue::String(text) => Ok(text), + _ => Err(NativeCompilerError::new_err(SURROGATE_KEY)), + }; + } + self.orjson_rejects(ORJSON_KEY.to_owned()); + if key.is_none() { + return Ok("null".to_owned()); + } + if let Ok(flag) = key.downcast::() { + return Ok(if flag.is_true() { "true" } else { "false" }.to_owned()); + } + if key.is_instance_of::() { + return Ok(key.str()?.to_str()?.to_owned()); + } + if key.is_instance_of::() { + return float_key(key); + } + let name: String = key.get_type().name()?.to_string(); + self.stdlib_rejects(PyTypeError::new_err(format!( + "keys must be str, int, float, bool or None, not {name}" + ))); + Ok(String::new()) + } +} + +/// `json.dumps` writes a float key as `float.__repr__`, or as `NaN` / `Infinity`. +fn float_key(key: &Bound<'_, PyAny>) -> PyResult { + let number: f64 = key.extract::()?; + if number.is_nan() { + return Ok("NaN".to_owned()); + } + if number.is_infinite() { + return Ok(if number.is_sign_positive() { + "Infinity" + } else { + "-Infinity" + } + .to_owned()); + } + Ok(key.repr()?.to_str()?.to_owned()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(emit_orjson_report, module)?)?; + module.add_function(wrap_pyfunction!(emit_json_report, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs new file mode 100644 index 0000000000..0afc71b85d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod artifacts; +pub(crate) mod json_report; +pub(crate) mod registration; +pub(crate) mod reuse; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs new file mode 100644 index 0000000000..4db4e556c6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs @@ -0,0 +1,11 @@ +//! Register the native compile output functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::compile_outputs::{artifacts, json_report, reuse}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + artifacts::register(module)?; + json_report::register(module)?; + reuse::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs new file mode 100644 index 0000000000..0abd4df25e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -0,0 +1,308 @@ +//! Whole-project compile reuse on the shared native store, driven by the Python host's facts. + +use std::collections::{BTreeMap, HashMap}; +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{FromPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_cache::digest::main::bytes_digest::bytes_digest; +use sqlbuild_cache::digest::main::hex_digest::hex_digest; +use sqlbuild_cache::project_reuse::constants::{NATIVE_DIGEST_PREFIX, REUSE_STORE_SUFFIX}; +use sqlbuild_cache::project_reuse::main::check_reuse::check_reuse; +use sqlbuild_cache::project_reuse::main::pending_digest_bytes::pending_digest_bytes; +use sqlbuild_cache::project_reuse::main::record_reuse::record_reuse; +use sqlbuild_cache::project_reuse::main::replay_stdout::replay_stdout; +use sqlbuild_cache::project_reuse::models::{ + CompileRecord, HostIdentity, RecordOutcome, RecordResult, RecordedArtifact, ReuseAttempt, + ReuseCheck, ReuseOutcome, ReuseRules, SettingsInputs, StoredOutput, +}; +use sqlbuild_cache::project_snapshot::main::path_text::path_text; +use sqlbuild_cache::project_snapshot::models::SnapshotRules; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +type SettingsRow = (bool, Vec, Vec, Vec, Vec); +type ReplayRow = (String, Vec, i64); +type LaunchIdentityRow = (String, BTreeMap, Vec); +/// A written artifact's digest, or a kept artifact's size and mtime. +type ArtifactRow = (Option, Option, Option); + +/// The host constants the walk, environment and store checks use. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct ReuseRulesPy { + excluded: Vec, + excluded_root: Vec, + presence_suffixes: Vec, + output_files: Vec<(u64, u64)>, + tracked_prefixes: Vec, + untracked_names: Vec, + disabling_variables: Vec<(String, String)>, + config_filenames: Vec, + racy_window_ns: i64, + max_stored_entries: usize, + max_entry_bytes: usize, + missing_environment_value: String, + missing_file_digest: String, + missing_path_mtime_ns: i64, + project_root_path_mtime_ns: i64, + compiled_root: PathBuf, + retired_directories: Vec, +} + +impl From for ReuseRules { + fn from(rules: ReuseRulesPy) -> Self { + Self { + snapshot: SnapshotRules { + excluded: rules.excluded, + excluded_root: rules.excluded_root, + presence_suffixes: rules.presence_suffixes, + output_files: rules.output_files, + }, + tracked_prefixes: rules.tracked_prefixes, + untracked_names: rules.untracked_names, + disabling_variables: rules.disabling_variables, + config_filenames: rules.config_filenames, + racy_window_ns: rules.racy_window_ns, + max_stored_entries: rules.max_stored_entries, + max_entry_bytes: rules.max_entry_bytes, + missing_environment_value: rules.missing_environment_value, + missing_file_digest: rules.missing_file_digest, + missing_path_mtime_ns: rules.missing_path_mtime_ns, + project_root_path_mtime_ns: rules.project_root_path_mtime_ns, + compiled_root: rules.compiled_root, + retired_directories: rules.retired_directories, + } + } +} + +/// One reuse check: where the project and its store slot are, and the host's identity facts. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct ReuseCheckPy { + project_dir: PathBuf, + store_directory: PathBuf, + selected_target: Option, + invocation: Vec, + runtime: BTreeMap, + search_path: Vec, + bypass_requested: bool, + json_output: bool, + rules: ReuseRulesPy, +} + +/// What a full compile leaves for recording; `artifacts` maps a path to `(digest, size, mtime)`. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct CompileRecordPy { + stdout: Option, + stderr_lines: Vec, + exit_code: i64, + compile_cache_enabled: bool, + read_run_id: bool, + settings_inputs: Option>, + template_environment_names: Vec, + module_paths: Vec, + artifacts: HashMap, + artifacts_written: bool, + dag_path: Option, + json_output: bool, + rules: ReuseRulesPy, +} + +/// One invocation's native reuse state, from the check to the record. +#[pyclass(module = "sqlbuild._native", frozen)] +struct NativeReuseAttempt { + attempt: Mutex, +} + +fn poisoned(_: T) -> pyo3::PyErr { + PyValueError::new_err("compile reuse attempt is poisoned") +} + +#[pymethods] +impl NativeReuseAttempt { + /// `"hit"`, `"miss"` or `"bypass"`. + fn outcome(&self) -> PyResult<&'static str> { + Ok(match self.attempt.lock().map_err(poisoned)?.outcome { + ReuseOutcome::Hit => "hit", + ReuseOutcome::Miss => "miss", + ReuseOutcome::Bypass => "bypass", + }) + } + + /// The store file holding this invocation's slot. + fn store_path(&self) -> PyResult { + let attempt = self.attempt.lock().map_err(poisoned)?; + Ok(attempt.store_path.to_string_lossy().into_owned()) + } + + /// Whether a stored compile matched but could not be replayed. + fn replay_failed(&self) -> PyResult { + Ok(self.attempt.lock().map_err(poisoned)?.replay_failed) + } + + /// How many project paths the check stamped. + fn snapshot_paths(&self) -> PyResult { + Ok(self.attempt.lock().map_err(poisoned)?.snapshot.len()) + } + + /// Bytes recording would hash for files whose stamp moved. + fn pending_digest_bytes(&self) -> PyResult { + let attempt = self.attempt.lock().map_err(poisoned)?; + Ok(pending_digest_bytes(&attempt)) + } + + /// On a hit, the stored report with these timings plus its stderr lines and exit code. + fn replay(&self, timings: Vec<(String, i64)>) -> PyResult> { + let attempt = self.attempt.lock().map_err(poisoned)?; + let Some(output) = attempt.replay.as_ref() else { + return Ok(None); + }; + let output: &StoredOutput = output; + Ok(replay_stdout(output, &timings) + .map(|stdout| (stdout, output.stderr_lines.clone(), output.exit_code))) + } + + /// The invocation digest, runtime and search path the check compared, for a launch record. + fn launch_identity(&self) -> PyResult { + let attempt = self.attempt.lock().map_err(poisoned)?; + let identity: &HostIdentity = &attempt.identity; + Ok(( + identity.invocation_digest.clone(), + identity.runtime.clone(), + identity.search_path.clone(), + )) + } + + /// How many project files the check read for content digests. + fn digested_files(&self) -> PyResult { + Ok(self.attempt.lock().map_err(poisoned)?.digested_files) + } + + /// Store this compile or clear the slot; a failure clears the slot and raises `OSError`. + fn record(&self, py: Python<'_>, record: CompileRecordPy) -> PyResult<(&'static str, usize)> { + compiler_guard(|| { + let rules: ReuseRules = record.rules.into(); + let compile: CompileRecord = CompileRecord { + stdout: record.stdout, + stderr_lines: record.stderr_lines, + exit_code: record.exit_code, + compile_cache_enabled: record.compile_cache_enabled, + read_run_id: record.read_run_id, + settings: record + .settings_inputs + .map(|rows| rows.into_iter().map(settings_inputs).collect()) + .ok_or_else(String::new), + template_environment_names: record.template_environment_names, + module_paths: texts(&record.module_paths), + artifacts: record + .artifacts + .into_iter() + .map(|(path, artifact)| (path, recorded_artifact(artifact))) + .collect(), + artifacts_written: record.artifacts_written, + dag_path: record.dag_path, + json_output: record.json_output, + }; + let attempt = self.attempt.lock().map_err(poisoned)?; + let result: RecordResult = py + .detach(|| record_reuse(&attempt, &compile, &rules)) + .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.message))?; + let outcome: &'static str = match result.outcome { + RecordOutcome::NotAttempted => "not_attempted", + RecordOutcome::Stored => "stored", + RecordOutcome::Removed => "removed", + RecordOutcome::Skipped => "skipped", + }; + Ok((outcome, result.digested_files)) + }) + } +} + +fn settings_inputs(row: SettingsRow) -> SettingsInputs { + let (case_sensitive, names, prefixes, env_files, secrets_dirs) = row; + SettingsInputs { + case_sensitive, + names, + prefixes, + env_files, + secrets_dirs, + } +} + +fn recorded_artifact(artifact: ArtifactRow) -> RecordedArtifact { + match artifact { + (Some(digest), _, _) => RecordedArtifact::Written { digest }, + (None, size, mtime_ns) => RecordedArtifact::Kept { + size: size.unwrap_or_default(), + mtime_ns: mtime_ns.unwrap_or_default(), + }, + } +} + +/// Check the stored compile for this invocation, replaying it on a hit. +#[pyfunction] +fn check_compile_reuse(py: Python<'_>, check: ReuseCheckPy) -> PyResult { + compiler_guard(|| { + let rules: ReuseRules = check.rules.into(); + let request: ReuseCheck<'_> = ReuseCheck { + project_dir: check.project_dir, + store_path: store_slot(&check.store_directory, check.selected_target.as_deref()), + identity: HostIdentity { + invocation_digest: native_digest(&check.invocation), + runtime: check + .runtime + .iter() + .map(|(name, value)| (name.clone(), path_text(value))) + .collect(), + search_path: texts(&check.search_path), + }, + bypass_requested: check.bypass_requested, + json_output: check.json_output, + rules: &rules, + }; + let attempt: ReuseAttempt = py.detach(|| check_reuse(request)); + Ok(NativeReuseAttempt { + attempt: Mutex::new(attempt), + }) + }) +} + +/// The native content digest of artifact bytes, as reuse verifies written artifacts. +#[pyfunction] +fn artifact_digest(contents: &[u8]) -> String { + native_digest(contents) +} + +fn native_digest(contents: &[u8]) -> String { + format!( + "{NATIVE_DIGEST_PREFIX}{}", + hex_digest(&bytes_digest(contents)) + ) +} + +/// The one store file kept for the selected target. +fn store_slot(directory: &Path, selected_target: Option<&OsStr>) -> PathBuf { + let label: String = selected_target.map_or_else(String::new, |target| { + format!("target:{}", path_text(target)) + }); + directory.join(format!( + "{}.{REUSE_STORE_SUFFIX}", + hex_digest(&bytes_digest(label.as_bytes())) + )) +} + +/// Host strings that may carry undecodable bytes, as exact text. +fn texts(values: &[OsString]) -> Vec { + values.iter().map(|value| path_text(value)).collect() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(check_compile_reuse, module)?)?; + module.add_function(wrap_pyfunction!(artifact_digest, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs new file mode 100644 index 0000000000..37618cd141 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs @@ -0,0 +1,763 @@ +//! `build_project_audit_inputs`: singular and attached generic audits attached natively. + +use std::collections::{BTreeMap, HashMap}; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods, +}; +use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge_call_site_references; +use sqlbuild_attachments::audits::main::attached_model_reference_error::attached_model_reference_error; +use sqlbuild_attachments::audits::main::audit_reference_error::audit_reference_error; +use sqlbuild_attachments::audits::main::render_attached_audit::render_attached_audit; +use sqlbuild_attachments::audits::main::resolve_audit_policies::resolve_audit_policies; +use sqlbuild_attachments::audits::main::singular_audit_resources::singular_audit_resources; +use sqlbuild_attachments::audits::models::{ + AuditAttachment, AuditRendering, PolicySource, SingularAuditResources, +}; +use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; +use sqlbuild_attachments::functions::models::KnownResources; +use sqlbuild_sqltext::sql_references::models::SqlReference; + +use crate::bindings::_helpers::attachments::argument_values::argument_pairs; +use crate::bindings::_helpers::compiled_project::audit_views::{AttachedAudit, AttachedTarget}; +use crate::bindings::_helpers::compiled_project::model_loop::{ + LoopContext, NativeModelLoop, compile_error, helped_compile_error, +}; +use crate::bindings::_helpers::compiled_project::model_loop_calls::origin_references; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::model_loop_inputs::reference_row; +use crate::bindings::_helpers::compiled_project::model_loop_named::{ + AUDIT_DECLARATION, resource_named_usages, +}; +use crate::bindings::_helpers::compiled_project::source_loop::{ + AuthoredExpansion, expanded_authored_sql, file_declarations, +}; +use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::header_metadata::thresholds; +use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::header_metadata::main::parse_measurement_policy::parse_measurement_policy; + +const THRESHOLDS_KEY: &str = "thresholds"; +const MINIMUM_SAMPLES_KEY: &str = "minimum_samples"; +const MEASUREMENT_HEADER_KEYS: [&str; 2] = [THRESHOLDS_KEY, MINIMUM_SAMPLES_KEY]; +const STANDALONE_MEASUREMENT_LABEL: &str = "standalone measurement audit"; +const SEVERITY_KEY: &str = "severity"; +const RUN_SCOPE_KEY: &str = "run_scope"; +const ALWAYS_RUN_KEY: &str = "always_run"; +const COLUMN_ARGUMENT: &str = "column"; +const RELATION_ARGUMENT: &str = "relation"; +const SINGULAR_AUDIT_CODE: &str = "P004"; +const BUILT_IN_SHADOW_CODE: &str = "P003"; +const AUDIT_RESOURCE_TYPE: &str = "audit"; +const MODEL_TARGET: &str = "model"; +const SOURCE_TARGET: &str = "source"; +const SEED_TARGET: &str = "seed"; + +/// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. +pub(crate) type AuditBlockRow = ( + Py, + Option, + String, + Option, + bool, + Py, +); + +/// An audit file's paths, stem, discovered file, generic flag, contents and blocks. +pub(crate) type AuditFileRow = ( + String, + String, + String, + Py, + bool, + String, + Vec, +); + +/// What every audit in one compile resolves against, and the Python values it builds with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct AuditSettings { + default_severity: Option, + default_run_scope: Option, + /// Built-in generic definitions as audit file rows, overridden by project definitions. + builtins: Vec, + /// `DeclarationIdentity`, and the singular and generic audit `DeclarationKind`s. + declaration_identity: Py, + singular_kind: Py, + audit_kind: Py, + /// `ResourceKind` by value, for the resources singular audits read and attach to. + resource_kinds: Py, +} + +/// Generic definitions by name, and the warnings for project ones overriding built-ins. +type Definitions<'a, 'py> = (HashMap>, Vec>); + +/// One generic definition: its file row and its only block. +struct Definition<'a> { + file: &'a AuditFileRow, + block: &'a AuditBlockRow, +} + +/// One resource's attached audits: its target, consumer path, and owner files and instances. +pub(crate) struct AttachmentOwner<'py> { + pub(crate) kind: &'static str, + pub(crate) name: String, + pub(crate) relation: String, + pub(crate) path: String, + /// `(owner file, column, instance)` in attachment order. + pub(crate) instances: Vec<(String, Option, Bound<'py, PyAny>)>, +} + +/// Every singular audit, then each owner's attached audits, and the built-in shadow warnings. +pub(crate) fn attached_audits<'py>( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, 'py>, + inputs: (&[AuditFileRow], Vec>), + settings: &AuditSettings, +) -> PyResult<(Vec, Vec>)> { + let (files, owners) = inputs; + let (definitions, shadows) = generic_definitions(context, files, &settings.builtins)?; + let scope = AuditScope { + model_loop, + context, + settings, + }; + let mut audits: Vec = Vec::new(); + for file in files.iter().filter(|file| !file.4) { + for block in &file.6 { + audits.push(scope.singular(file, block)?); + } + } + for owner in &owners { + for (owner_file, column, instance) in &owner.instances { + audits.push(scope.attached( + &definitions, + owner, + (owner_file, column.as_deref()), + instance, + )?); + } + } + Ok((audits, shadows)) +} + +/// `index_generic_audit_definitions` merged over the built-ins, with P003 for each override. +fn generic_definitions<'a, 'py>( + context: &LoopContext<'_, 'py>, + files: &'a [AuditFileRow], + builtins: &'a [AuditFileRow], +) -> PyResult> { + let py: Python<'py> = context.host.py(); + let mut project: BTreeMap> = BTreeMap::new(); + for file in files.iter().filter(|file| file.4) { + let [block] = file.6.as_slice() else { + let message: String = format!( + "Generic audit definition {} must contain exactly one AUDIT block", + file.1 + ); + return Err(compile_error(context.classes, &message, None)); + }; + let header: &Bound<'_, PyDict> = block.5.bind(py); + if block.4 + && (header.contains(MEASUREMENT_HEADER_KEYS[0])? + || header.contains(MEASUREMENT_HEADER_KEYS[1])?) + { + let message: String = format!( + "Generic measurement audit definition {} must not define thresholds or \ + minimum_samples in its AUDIT header; attachment owns policy", + file.1 + ); + return Err(compile_error(context.classes, &message, None)); + } + if let Some(existing) = project.get(&file.2) { + let message: String = format!( + "Duplicate generic audit definition found for '{}' in {} and {}", + file.2, existing.file.1, file.1 + ); + return Err(compile_error(context.classes, &message, None)); + } + project.insert(file.2.clone(), Definition { file, block }); + } + let mut shadows: Vec> = Vec::new(); + let mut definitions: HashMap> = HashMap::new(); + for builtin in builtins { + if let Some(block) = builtin.6.first() { + definitions.insert( + builtin.2.clone(), + Definition { + file: builtin, + block, + }, + ); + } + } + for (name, definition) in project { + if definitions.contains_key(&name) { + shadows.push(shadow_warning(context, &name, &definition.file.1)?); + } + definitions.insert(name, definition); + } + Ok((definitions, shadows)) +} + +/// The P003 warning for a project audit that overrides a built-in one. +fn shadow_warning<'py>( + context: &LoopContext<'_, 'py>, + name: &str, + relative_path: &str, +) -> PyResult> { + let py: Python<'py> = context.host.py(); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", "warning")?; + fields.set_item("code", BUILT_IN_SHADOW_CODE)?; + fields.set_item( + "message", + format!("project audit '{name}' overrides built-in audit '{name}'"), + )?; + fields.set_item("path", context.classes.path_class.call1((relative_path,))?)?; + fields.set_item( + "help", + "rename the project audit if this override is unintended", + )?; + context.classes.compiler_diagnostic.call((), Some(&fields)) +} + +/// The loop, its context and the settings each audit is attached with. +struct AuditScope<'a, 'py> { + model_loop: &'a NativeModelLoop, + context: &'a LoopContext<'a, 'py>, + settings: &'a AuditSettings, +} + +/// An audit's expanded SQL bodies, references and usages. +struct ExpandedAudit<'py> { + sql_body: String, + evidence_sql: Option, + references: Vec, + usages: Vec>, +} + +impl<'py> AuditScope<'_, 'py> { + /// One singular audit block, as `_build_singular_audit_inputs` builds it. + fn singular(&self, file: &AuditFileRow, block: &AuditBlockRow) -> PyResult { + let py: Python<'py> = self.context.host.py(); + let (_, relative_path, stem, audit_file, _, _, _) = file; + let (audit_block, block_name, sql_body, evidence_sql, measurement, header) = block; + let audit_name: String = block_name.clone().unwrap_or_else(|| stem.clone()); + let identity: Bound<'py, PyAny> = self + .settings + .declaration_identity + .bind(py) + .call1((self.settings.singular_kind.bind(py), &audit_name))?; + let label: String = format!("Audit '{audit_name}'"); + let expanded: ExpandedAudit<'py> = + self.expanded(file, (sql_body, evidence_sql.as_deref()), &identity, &label)?; + let calls: Vec<(&str, &str)> = reference_calls(&expanded.references); + self.checked_references(relative_path, &calls)?; + let resources: SingularAuditResources = singular_audit_resources(&calls); + if let Some((message, help)) = &resources.problem { + self.report_not_cross_resource(relative_path, &audit_name, (message, help))?; + } + let header: &Bound<'py, PyDict> = header.bind(py); + let severity: Option = header_text(header, SEVERITY_KEY)?; + let run_scope: Option = header_text(header, RUN_SCOPE_KEY)?; + let always_run: bool = match header.get_item(ALWAYS_RUN_KEY)? { + Some(value) => value.downcast::().is_ok_and(|flag| flag.is_true()), + None => false, + }; + let (thresholds, minimum_samples) = self.measurement_policy(header, relative_path)?; + let (severity, run_scope) = if *measurement { + if severity.is_some() { + let message: String = format!( + "{relative_path}: measurement audits must not define severity; severity \ + derives from thresholds" + ); + return Err(compile_error(self.context.classes, &message, None)); + } + if thresholds.is_none() { + let message: String = format!( + "{relative_path}: standalone measurement audit must define thresholds in its \ + AUDIT header" + ); + return Err(compile_error(self.context.classes, &message, None)); + } + let severity: &'static str = if thresholds.getattr("error")?.is_none() { + "warn" + } else { + "error" + }; + ( + severity, + self.policies(None, relative_path, run_scope.as_deref())?.1, + ) + } else { + self.policies( + Some(severity.as_deref()), + relative_path, + run_scope.as_deref(), + )? + }; + let mut usages: Vec> = expanded.usages; + let resource_kinds: &Bound<'py, PyDict> = self.settings.resource_kinds.bind(py); + for (kind, name) in &resources.resources { + let kind: Bound<'py, PyAny> = resource_kinds + .get_item(kind)? + .ok_or_else(|| compile_error(self.context.classes, kind, None))?; + let resource: Bound<'py, PyAny> = self + .context + .classes + .visibility + .resource_identity + .call1((kind, name))?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("consumer", resource)?; + fields.set_item("declaration", &identity)?; + usages.push(self.context.classes.usage_record.call((), Some(&fields))?); + } + Ok(AttachedAudit { + audit_file: audit_file.clone_ref(py), + audit_block: audit_block.clone_ref(py), + definition_name: audit_name, + sql_body: expanded.sql_body, + measurement: *measurement, + evidence_sql: expanded.evidence_sql, + thresholds: thresholds.unbind(), + minimum_samples: minimum_samples.unbind(), + evidence_limit: py.None(), + references: expanded.references, + target: None, + severity, + run_scope, + always_run, + usages: PyTuple::new(py, usages)?.unbind(), + name: None, + description: None, + }) + } + + /// One attached generic audit instance, as `build_attached_audit_input` builds it. + fn attached( + &self, + definitions: &HashMap>, + owner: &AttachmentOwner<'py>, + attachment: (&str, Option<&str>), + instance: &Bound<'py, PyAny>, + ) -> PyResult { + let py: Python<'py> = self.context.host.py(); + let (owner_file, column) = attachment; + let definition_name: String = instance.getattr("definition_name")?.extract()?; + let Some(definition) = definitions.get(&definition_name) else { + let message: String = + format!("{owner_file} references unknown generic audit '{definition_name}'"); + return Err(compile_error(self.context.classes, &message, None)); + }; + let consumer: Bound<'py, PyAny> = + self.context.classes.visibility.resource_identity.call1(( + self.settings.resource_kinds.bind(py).get_item(owner.kind)?, + &owner.name, + ))?; + let mut usages: Vec> = resource_named_usages( + self.context, + (&AUDIT_DECLARATION, &definition_name), + &consumer, + (owner.kind, &owner.name, &owner.path), + )?; + let thresholds: Bound<'py, PyAny> = instance.getattr("thresholds")?; + let minimum_samples: Bound<'py, PyAny> = instance.getattr("minimum_samples")?; + let instance_severity: Option = instance.getattr("severity")?.extract()?; + let label: String = format!("{owner_file} audit '{definition_name}'"); + let measurement: bool = definition.block.4; + if measurement { + if instance_severity.is_some() { + let message: String = format!( + "{label}: measurement audit attachments must not define severity; severity \ + derives from thresholds" + ); + return Err(compile_error(self.context.classes, &message, None)); + } + if thresholds.is_none() { + let message: String = format!( + "{label}: measurement audit attachments must define at least one threshold" + ); + return Err(compile_error(self.context.classes, &message, None)); + } + } else if !thresholds.is_none() || !minimum_samples.is_none() { + let message: String = format!( + "{label}: thresholds and minimum_samples are only valid for measurement audits" + ); + return Err(compile_error(self.context.classes, &message, None)); + } + let mut implicit: Vec<(String, String)> = vec![ + (owner.kind.to_owned(), owner.name.clone()), + (RELATION_ARGUMENT.to_owned(), owner.relation.clone()), + ]; + if let Some(column) = column { + implicit.push((COLUMN_ARGUMENT.to_owned(), column.to_owned())); + } + let rendering: AuditRendering = render_attached_audit(&AuditAttachment { + owner_label: owner_file.to_owned(), + definition_name: definition_name.clone(), + sql_body: definition.block.2.clone(), + evidence_sql: definition.block.3.clone(), + implicit_arguments: implicit, + explicit_arguments: argument_pairs(instance.getattr("arguments")?.downcast()?)?, + has_thresholds: !thresholds.is_none(), + threshold_error: !thresholds.is_none() && !thresholds.getattr("error")?.is_none(), + instance_severity, + default_severity: self.settings.default_severity.clone(), + instance_run_scope: instance.getattr("run_scope")?.extract()?, + default_run_scope: self.settings.default_run_scope.clone(), + }); + let rendered = match rendering { + AuditRendering::Failed(message) => { + return Err(compile_error(self.context.classes, &message, None)); + } + AuditRendering::Rendered(rendered) => rendered, + }; + let identity: Bound<'py, PyAny> = self + .settings + .declaration_identity + .bind(py) + .call1((self.settings.audit_kind.bind(py), &definition_name))?; + let expanded: ExpandedAudit<'py> = self.expanded( + definition.file, + (&rendered.sql_body, rendered.evidence_sql.as_deref()), + &identity, + &format!("Audit '{definition_name}'"), + )?; + let calls: Vec<(&str, &str)> = reference_calls(&expanded.references); + self.checked_references(&definition.file.1, &calls)?; + let (severity, source) = rendered + .policies + .map_err(|message| compile_error(self.context.classes, &message, None))?; + let run_scope: String = match source { + PolicySource::Instance => instance.getattr("run_scope")?.extract()?, + PolicySource::Default => self.settings.default_run_scope.clone().unwrap_or_default(), + PolicySource::Fallback => "delta_and_final".to_owned(), + }; + if let Some(message) = + attached_model_reference_error(&label, (owner.kind, &owner.name), &calls) + { + return Err(compile_error(self.context.classes, &message, None)); + } + usages.extend(expanded.usages); + Ok(AttachedAudit { + audit_file: definition.file.3.clone_ref(py), + audit_block: definition.block.0.clone_ref(py), + definition_name: definition + .block + .1 + .clone() + .unwrap_or_else(|| definition.file.2.clone()), + sql_body: expanded.sql_body, + measurement, + evidence_sql: expanded.evidence_sql, + thresholds: thresholds.unbind(), + minimum_samples: minimum_samples.unbind(), + evidence_limit: instance.getattr("evidence_limit")?.unbind(), + references: expanded.references, + target: Some(AttachedTarget { + kind: owner.kind.to_owned(), + name: owner.name.clone(), + column: column.map(str::to_owned), + }), + severity, + run_scope, + always_run: instance.getattr(ALWAYS_RUN_KEY)?.is_truthy()?, + usages: PyTuple::new(py, usages)?.unbind(), + name: instance.getattr("name")?.extract()?, + description: instance.getattr("description")?.extract()?, + }) + } + + /// Body and evidence expanded in `file`'s scope, intrinsic-free, with their references. + fn expanded( + &self, + file: &AuditFileRow, + sql: (&str, Option<&str>), + identity: &Bound<'py, PyAny>, + label: &str, + ) -> PyResult> { + let (file_path, relative_path, _, _, _, contents, _) = file; + let declarations: Bound<'py, PyAny> = file_declarations(self.context, file_path, identity)?; + let body: AuthoredExpansion<'py> = expanded_authored_sql( + self.model_loop, + self.context, + (sql.0, file_path), + &declarations, + )?; + let evidence: Option> = sql + .1 + .map(|evidence| { + expanded_authored_sql( + self.model_loop, + self.context, + (evidence, file_path), + &declarations, + ) + }) + .transpose()?; + self.intrinsic_free(&body.sql, label)?; + if let Some(evidence) = &evidence { + self.intrinsic_free(&evidence.sql, &format!("{label} evidence"))?; + } + let mut scanned: Vec = + self.references(&body.sql, (file_path, relative_path, contents))?; + let mut arguments: Vec = body.argument_references; + let mut usages: Vec> = body.usages.iter().collect(); + if let Some(evidence) = &evidence { + for row in self.references(&evidence.sql, (file_path, relative_path, contents))? { + if !scanned.contains(&row) { + scanned.push(row); + } + } + arguments.extend(evidence.argument_references.iter().cloned()); + usages.extend(evidence.usages.iter()); + } + Ok(ExpandedAudit { + sql_body: body.sql, + evidence_sql: evidence.map(|evidence| evidence.sql), + references: merge_call_site_references(scanned, arguments), + usages, + }) + } + + fn intrinsic_free(&self, sql: &str, label: &str) -> PyResult<()> { + match intrinsic_free( + self.model_loop.python, + sql, + &self.model_loop.reserved_markers, + label, + ) { + IntrinsicCheck::Free => Ok(()), + IntrinsicCheck::Rejected(message) => { + Err(compile_error(self.context.classes, &message, None)) + } + } + } + + /// One SQL text's references, reporting rejected calls against the audit file. + fn references(&self, sql: &str, file: (&str, &str, &str)) -> PyResult> { + let references: Vec = + origin_references(self.context, &self.model_loop.syntax, sql, file)?; + let mut rows: Vec = Vec::with_capacity(references.len()); + for row in references.iter().map(reference_row) { + if !rows.contains(&row) { + rows.push(row); + } + } + Ok(rows) + } + + /// `validate_audit_references` against the project's discovered resources. + fn checked_references(&self, relative_path: &str, calls: &[(&str, &str)]) -> PyResult<()> { + let validator: Bound<'_, NativeModelValidator> = self + .model_loop + .validator + .bind(self.context.host.py()) + .downcast()? + .clone(); + let known: KnownResources<'_> = validator.get().known_resources(); + match audit_reference_error(relative_path, calls, &known) { + Some(message) => Err(compile_error(self.context.classes, &message, None)), + None => Ok(()), + } + } + + /// The header's `MeasurementThresholds` and authored `minimum_samples`, as Python parses them. + fn measurement_policy( + &self, + header: &Bound<'py, PyDict>, + relative_path: &str, + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = header.py(); + let raw = + |key: &str| -> PyResult>> { Ok(header.get_item(key)?.map(PyNode)) }; + let (thresholds_node, samples_node) = (raw(THRESHOLDS_KEY)?, raw(MINIMUM_SAMPLES_KEY)?); + let (parsed, minimum_samples) = parse_measurement_policy( + (thresholds_node.as_ref(), samples_node.as_ref()), + relative_path, + STANDALONE_MEASUREMENT_LABEL, + ) + .map_err(|error| self.config_error(&error))?; + Ok(( + thresholds(py, self.context.contracts, parsed.as_ref())?, + minimum_samples.map_or_else(|| py.None().into_bound(py), |node| node.0), + )) + } + + /// `CompileInputError` with `error`'s message and help. + fn config_error(&self, error: &ConfigError) -> PyErr { + helped_compile_error(self.context.classes, &error.message, error.help.as_deref()) + } + + /// The resolved severity and run scope; a measurement (`None`) severity reads no default. + fn policies( + &self, + severity: Option>, + label: &str, + run_scope: Option<&str>, + ) -> PyResult<(&'static str, String)> { + let (instance, default) = match severity { + Some(instance) => (instance, self.settings.default_severity.as_deref()), + None => (None, None), + }; + resolve_audit_policies( + (instance, default, label), + (run_scope, self.settings.default_run_scope.as_deref()), + ) + .map_err(|message| compile_error(self.context.classes, &message, None)) + } + + /// Report P004 for a singular audit that does not read two resources. + fn report_not_cross_resource( + &self, + relative_path: &str, + audit_name: &str, + problem: (&str, &str), + ) -> PyResult<()> { + let py: Python<'py> = self.context.host.py(); + let (message, help) = problem; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", "error")?; + fields.set_item("code", SINGULAR_AUDIT_CODE)?; + fields.set_item( + "message", + format!("Singular audit '{audit_name}' in {relative_path} {message}"), + )?; + fields.set_item("resource_type", AUDIT_RESOURCE_TYPE)?; + fields.set_item("resource_name", audit_name)?; + fields.set_item( + "path", + self.context.classes.path_class.call1((relative_path,))?, + )?; + fields.set_item("help", help)?; + let diagnostic: Bound<'py, PyAny> = self + .context + .classes + .compiler_diagnostic + .call((), Some(&fields))?; + let key: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + SINGULAR_AUDIT_CODE, + &relative_path.replace('\\', "/"), + audit_name, + ], + )?; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("key", key)?; + arguments.set_item("diagnostic", diagnostic)?; + self.context + .classes + .report_diagnostic + .call((), Some(&arguments))?; + Ok(()) + } +} + +fn reference_calls(references: &[ReferenceRow]) -> Vec<(&str, &str)> { + references + .iter() + .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) + .collect() +} + +/// `_str_from_dict`: a header value when it is a string. +fn header_text(header: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match header.get_item(key)? { + Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} + +/// The attachment owners of every model, source and seed, as the audit loop reads them. +pub(crate) fn model_owner<'py>( + name: String, + path: String, + schema: (&Bound<'py, PyTuple>, &Bound<'py, PyTuple>), +) -> PyResult> { + let (audits, columns) = schema; + let mut instances: Vec<(String, Option, Bound<'py, PyAny>)> = Vec::new(); + for audit in audits.iter() { + instances.push((path.clone(), explicit_column(&audit)?, audit)); + } + for column in columns.iter() { + let column_owner: String = location_path(&column)?.unwrap_or_else(|| path.clone()); + let column_name: String = column.getattr("name")?.extract()?; + for audit in column.getattr("audits")?.try_iter()? { + let audit: Bound<'py, PyAny> = audit?; + let owner_file: String = location_path(&audit)?.unwrap_or_else(|| column_owner.clone()); + instances.push((owner_file, Some(column_name.clone()), audit)); + } + } + Ok(AttachmentOwner { + kind: MODEL_TARGET, + relation: format!("__ref(\"{}\")", name.replace('"', "\"\"")), + name, + path, + instances, + }) +} + +/// A source's or seed's attachment owner; source table audits name no explicit column. +pub(crate) fn resource_owner<'py>( + kind: &'static str, + name: String, + paths: (String, String), + schema: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let (path, owner_file) = paths; + let (audits, columns) = schema; + let mut instances: Vec<(String, Option, Bound<'py, PyAny>)> = Vec::new(); + for audit in audits.try_iter()? { + let audit: Bound<'py, PyAny> = audit?; + let column: Option = if kind == SEED_TARGET { + explicit_column(&audit)? + } else { + None + }; + instances.push((owner_file.clone(), column, audit)); + } + for column in columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let column_name: String = column.getattr("name")?.extract()?; + for audit in column.getattr("audits")?.try_iter()? { + instances.push((owner_file.clone(), Some(column_name.clone()), audit?)); + } + } + let function: &str = if kind == SOURCE_TARGET { + "__source" + } else { + "__seed" + }; + Ok(AttachmentOwner { + kind, + relation: format!("{function}(\"{}\")", name.replace('"', "\"\"")), + name, + path, + instances, + }) +} + +/// `_explicit_audit_column_name`: the instance's `column` argument when it is a string. +fn explicit_column(audit: &Bound<'_, PyAny>) -> PyResult> { + let arguments: Bound<'_, PyDict> = audit.getattr("arguments")?.downcast_into()?; + header_text(&arguments, COLUMN_ARGUMENT) +} + +/// The path of a spec's authored location, if it has one. +fn location_path(spec: &Bound<'_, PyAny>) -> PyResult> { + let location: Bound<'_, PyAny> = spec.getattr("location")?; + if location.is_none() { + return Ok(None); + } + Ok(Some(location.getattr("path")?.str()?.to_string())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs new file mode 100644 index 0000000000..3f7ea723c5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs @@ -0,0 +1,394 @@ +//! Audits attached by the native loop, held by the compiled project, and their views. + +use pyo3::exceptions::{PyIndexError, PyRuntimeError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyTuple; +use pyo3::{FromPyObject, pyclass, pymethods}; + +use crate::bindings::_helpers::compiled_project::compiled_model_view::object_key; +use crate::bindings::_helpers::compiled_project::function_views::ObjectKeyRow; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + InputViewClasses, reference_tuple, view_classes, +}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; + +const AUDIT_RESOURCE: &str = "audit"; +const AUDIT_CLASSES: &str = "audits"; +const NOT_ASSEMBLED: &str = "the audit was not assembled"; + +/// The Python enums an audit view projects its severity and attached target kind with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct AuditClasses<'py> { + audit_severity: Bound<'py, PyAny>, + attached_target_kind: Bound<'py, PyAny>, +} + +/// `(references as (kind, name, package), attached (kind, name))`. +pub(crate) type AuditAssemblyRow = ( + Vec<(String, String, Option)>, + Option<(String, String)>, +); + +/// The target an attached audit checks: its kind, name and column. +pub(crate) struct AttachedTarget { + pub(crate) kind: String, + pub(crate) name: String, + pub(crate) column: Option, +} + +/// What the audit loop attached for one singular audit block or attached generic audit. +pub(crate) struct AttachedAudit { + /// The discovered (or built-in) audit file and block the SQL comes from. + pub(crate) audit_file: Py, + pub(crate) audit_block: Py, + /// The block's name, else the file's stem. + pub(crate) definition_name: String, + pub(crate) sql_body: String, + pub(crate) measurement: bool, + pub(crate) evidence_sql: Option, + /// The authored measurement policy specs, carrying user values. + pub(crate) thresholds: Py, + pub(crate) minimum_samples: Py, + pub(crate) evidence_limit: Py, + pub(crate) references: Vec, + pub(crate) target: Option, + pub(crate) severity: &'static str, + pub(crate) run_scope: String, + pub(crate) always_run: bool, + /// Usage records, the identities the scope index is built from. + pub(crate) usages: Py, + pub(crate) name: Option, + pub(crate) description: Option, +} + +impl AttachedAudit { + /// `_resolve_audit_name`: the instance name, else the definition name. + fn compiled_name(&self) -> &str { + self.name.as_deref().unwrap_or(&self.definition_name) + } +} + +/// One attached audit held by the compiled project. +pub(crate) struct AuditRecord { + pub(crate) attached: AttachedAudit, + /// The scope dependency keys native project assembly resolved, once it ran. + scope_deps: Option>, + references: PyOnceLock>, + key: PyOnceLock>, +} + +impl AuditRecord { + pub(crate) fn new(attached: AttachedAudit) -> Self { + Self { + attached, + scope_deps: None, + references: PyOnceLock::new(), + key: PyOnceLock::new(), + } + } + + /// Keep the scope dependency keys native project assembly resolved for the audit. + pub(crate) fn set_scope_deps(&mut self, deps: Vec) { + self.scope_deps = Some(deps); + } +} + +fn read_audit<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&AuditRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + let audit: &AuditRecord = owner + .audits() + .get(index) + .ok_or_else(|| PyIndexError::new_err(index))?; + read(audit, py) +} + +fn audit_classes<'py>( + owner: &Py, + py: Python<'py>, +) -> PyResult> { + owner + .bind(py) + .borrow() + .input_classes(py)? + .get_item(AUDIT_CLASSES)? + .extract() +} + +/// The attributes an audit's input and compiled views share. +macro_rules! audit_getters { + ($view:ty, { $($extra:tt)* }) => { + #[pymethods] + impl $view { + $($extra)* + + #[getter] + fn audit_file(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, py| { + Ok(audit.attached.audit_file.clone_ref(py)) + }) + } + + #[getter] + fn audit_block(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, py| { + Ok(audit.attached.audit_block.clone_ref(py)) + }) + } + + #[getter] + fn sql_body(&self, py: Python<'_>) -> PyResult { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit.attached.sql_body.clone()) + }) + } + + #[getter] + fn evaluation_mode(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, py| { + Ok(audit + .attached + .audit_block + .bind(py) + .getattr("evaluation_mode")? + .unbind()) + }) + } + + #[getter] + fn measurement_contract(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, py| { + Ok(audit + .attached + .audit_block + .bind(py) + .getattr("measurement_contract")? + .unbind()) + }) + } + + #[getter] + fn thresholds(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, py| { + Ok(audit.attached.thresholds.clone_ref(py)) + }) + } + + #[getter] + fn minimum_samples(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, py| { + Ok(audit.attached.minimum_samples.clone_ref(py)) + }) + } + + #[getter] + fn measure_sql(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit + .attached + .measurement + .then(|| audit.attached.sql_body.clone())) + }) + } + + #[getter] + fn evidence_sql(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit.attached.evidence_sql.clone()) + }) + } + + #[getter] + fn evidence_limit(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, py| { + Ok(audit.attached.evidence_limit.clone_ref(py)) + }) + } + + #[getter] + fn references(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_audit(&self.owner, self.index, py, |audit, _| { + reference_tuple(&classes, &audit.attached.references, &audit.references) + }) + } + + #[getter] + fn attached_target_kind<'py>(&self, py: Python<'py>) -> PyResult> { + let classes: AuditClasses<'py> = audit_classes(&self.owner, py)?; + read_audit(&self.owner, self.index, py, |audit, py| { + match &audit.attached.target { + Some(target) => classes.attached_target_kind.call1((&target.kind,)), + None => Ok(py.None().into_bound(py)), + } + }) + } + + #[getter] + fn attached_target_name(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit.attached.target.as_ref().map(|target| target.name.clone())) + }) + } + + #[getter] + fn attached_column_name(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit + .attached + .target + .as_ref() + .and_then(|target| target.column.clone())) + }) + } + + #[getter] + fn severity<'py>(&self, py: Python<'py>) -> PyResult> { + let classes: AuditClasses<'py> = audit_classes(&self.owner, py)?; + read_audit(&self.owner, self.index, py, |audit, _| { + classes.audit_severity.call1((audit.attached.severity,)) + }) + } + + #[getter] + fn run_scope(&self, py: Python<'_>) -> PyResult { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit.attached.run_scope.clone()) + }) + } + + #[getter] + fn always_run(&self, py: Python<'_>) -> PyResult { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit.attached.always_run) + }) + } + + #[getter] + fn description(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit.attached.description.clone()) + }) + } + } + }; +} + +/// One attached audit, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeAuditInput { + owner: Py, + index: usize, +} + +impl NativeAuditInput { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +audit_getters!(NativeAuditInput, { + #[getter] + fn declaration_usages(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, py| { + Ok(audit.attached.usages.clone_ref(py)) + }) + } + + #[getter] + fn name(&self, py: Python<'_>) -> PyResult> { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit.attached.name.clone()) + }) + } + + /// `(references as (kind, name, package), attached (kind, name))` for project assembly. + #[getter] + fn assembly_row(&self, py: Python<'_>) -> PyResult { + read_audit(&self.owner, self.index, py, |audit, _| { + let attached: &AttachedAudit = &audit.attached; + Ok(( + attached + .references + .iter() + .map(|(kind, name, package, _)| (kind.clone(), name.clone(), package.clone())) + .collect(), + attached + .target + .as_ref() + .map(|target| (target.kind.clone(), target.name.clone())), + )) + }) + } +}); + +/// One assembled audit, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeCompiledAudit { + owner: Py, + index: usize, +} + +impl NativeCompiledAudit { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +audit_getters!(NativeCompiledAudit, { + /// The audit's planner-facing `CompiledObjectKey`. + #[getter] + fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_audit(&self.owner, self.index, py, |audit, py| { + audit + .key + .get_or_try_init(py, || { + object_key( + &classes.compiled, + AUDIT_RESOURCE, + audit.attached.compiled_name(), + ) + }) + .map(|key| key.clone_ref(py)) + }) + } + + /// The planner-facing keys of the resources the audit is selected with. + #[getter] + fn scope_deps<'py>(&self, py: Python<'py>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_audit(&self.owner, self.index, py, |audit, py| { + let deps: &[ObjectKeyRow] = audit + .scope_deps + .as_deref() + .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; + let mut keys: Vec> = Vec::with_capacity(deps.len()); + for (kind, name) in deps { + keys.push(object_key(&classes.compiled, kind, name)?); + } + PyTuple::new(py, keys) + }) + } + + #[getter] + fn name(&self, py: Python<'_>) -> PyResult { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit.attached.compiled_name().to_owned()) + }) + } + + #[getter] + fn definition_name(&self, py: Python<'_>) -> PyResult { + read_audit(&self.owner, self.index, py, |audit, _| { + Ok(audit.attached.definition_name.clone()) + }) + } +}); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs new file mode 100644 index 0000000000..a14461c56c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -0,0 +1,385 @@ +//! A compiled model's analysis, kept from the native session and projected on demand. + +use std::collections::HashMap; + +use pyo3::IntoPyObject; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; +use sqlbuild_analysis::assembly::analysis_session::models::{ + ColumnFact, ContractProof, LineageFacts, LineageRow, ModelOutcome, PivotOutcome, +}; +use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; +use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; +use sqlbuild_analysis::semantic_validation::types::Expansion; + +use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + CompiledViewClasses, InferredRow, +}; +use crate::bindings::_helpers::compiled_project::model_input_view::ModelInputRecord; + +const COMPILE_PHASE: &str = "compile"; +const MODEL_RESOURCE: &str = "model"; + +/// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. +pub(crate) type ProofArgument = ( + bool, + Vec, + Vec<(String, Option)>, + Vec, + Option, + bool, +); + +/// The session analysis a model was assembled from: the session and its request position. +pub(crate) type SessionArgument = (Py, usize); + +/// What one model's analysis left for its compiled model. +pub(crate) struct AnalysisRecord { + pub(crate) inferred: Option>, + lineage: Option, + pub(crate) has_star: bool, + pub(crate) star_resolved: bool, + binding: Vec, + pub(crate) binding_validated: bool, + pub(crate) proof: Option, + projections: AnalysisProjections, +} + +/// Python projections of the analysis, each built once on first access. +struct AnalysisProjections { + lineage: PyOnceLock>, + binding: PyOnceLock>, + proof: PyOnceLock>, +} + +impl Default for AnalysisProjections { + fn default() -> Self { + Self { + lineage: PyOnceLock::new(), + binding: PyOnceLock::new(), + proof: PyOnceLock::new(), + } + } +} + +fn inferred_row(column: &ColumnFact) -> InferredRow { + ( + column.name.clone(), + column.data_type.clone(), + column.nullability.clone(), + ) +} + +fn proof_fact(argument: ProofArgument) -> ContractProof { + let (output_proven, fixed, families, input_relations, failure_reason, bare_dynamic_pivot) = + argument; + ContractProof { + output_proven, + fixed_columns: fixed + .into_iter() + .map(|(name, data_type, nullability)| ColumnFact { + name, + data_type, + nullability, + }) + .collect(), + families, + input_relations, + failure_reason, + bare_dynamic_pivot, + } +} + +/// The model's analysis: the session's outcome, a failed analysis's fallback columns, the proof. +pub(crate) fn analysis_record( + py: Python<'_>, + input: &ModelInputRecord, + analysis: Option, + (fallback, standalone_proof): (Option>, Option), +) -> PyResult { + let mut record: AnalysisRecord = AnalysisRecord { + inferred: None, + lineage: None, + has_star: false, + star_resolved: false, + binding: Vec::new(), + binding_validated: false, + proof: standalone_proof.map(proof_fact), + projections: AnalysisProjections::default(), + }; + if let Some((session, index)) = analysis { + let session = session.borrow(py); + let (outcome, contract): &(ModelOutcome, PivotOutcome) = session.outcome(index)?; + let analysed = &outcome.analysis; + if analysed.analysis_succeeded { + record.inferred = analysed + .columns + .as_ref() + .map(|columns| columns.iter().map(inferred_row).collect()); + record.lineage = Some(analysed.lineage.clone()); + record.has_star = analysed.has_star; + record.star_resolved = analysed.star_resolved; + } else { + record.inferred = fallback; + } + if let PivotOutcome::Proof(proof) = contract { + record.proof = Some(proof.clone()); + } + record.binding = located( + py, + input, + &analysed.binding_diagnostics, + &outcome.cleaned_sql, + )?; + record.binding_validated = analysed.binding_validated; + } + if let Some(proof) = record.proof.as_ref().filter(|proof| proof.output_proven) { + record.inferred = Some(proof.fixed_columns.iter().map(inferred_row).collect()); + record.has_star = true; + } + Ok(record) +} + +/// The model's binding diagnostics at their authored locations. +fn located( + py: Python<'_>, + input: &ModelInputRecord, + rows: &[sqlbuild_analysis::semantic_validation::types::DiagnosticRow], + cleaned: &str, +) -> PyResult> { + if rows.is_empty() { + return Ok(Vec::new()); + } + let model_file: &Bound<'_, PyAny> = input.model_file.bind(py); + let authored: String = model_file.getattr("contents")?.extract()?; + let query: String = model_file.getattr("query_sql")?.extract()?; + let expansion = &input.expansion; + let passes: Vec> = vec![ + expansion.declaration_spans.clone(), + expansion.macro_spans.clone(), + ]; + let source: BindingSource<'_> = BindingSource { + authored: &authored, + query: &query, + cleaned, + expansion: expansion + .authored + .then_some((expansion.expanded_sql.bind(py).to_str()?, passes.as_slice())), + }; + locate_binding_diagnostics(rows, &source).map_err(PyValueError::new_err) +} + +impl AnalysisRecord { + /// The session's lineage as `CompactLineageFacts` or plain facts, or `None`. + pub(crate) fn lineage_object( + &self, + py: Python<'_>, + classes: &CompiledViewClasses<'_>, + ) -> PyResult> { + self.projections + .lineage + .get_or_try_init(py, || match &self.lineage { + None => Ok(py.None()), + Some(LineageFacts::Native(rows)) => compact_lineage(classes, rows), + Some(LineageFacts::NativeFacts(rows)) => lineage_facts(classes, rows), + }) + .map(|lineage| lineage.clone_ref(py)) + } + + /// The model's `CompilerDiagnostic` tuple, built once. + pub(crate) fn binding_tuple( + &self, + py: Python<'_>, + classes: &CompiledViewClasses<'_>, + model: (&str, &Bound<'_, PyAny>), + ) -> PyResult> { + self.projections + .binding + .get_or_try_init(py, || { + let mut diagnostics: Vec> = Vec::with_capacity(self.binding.len()); + for diagnostic in &self.binding { + diagnostics.push(compiler_diagnostic(classes, diagnostic, model)?); + } + Ok::<_, pyo3::PyErr>(PyTuple::new(py, diagnostics)?.unbind()) + }) + .map(|diagnostics| diagnostics.clone_ref(py)) + } + + /// The model's `DynamicColumnContractProof`, or `None`. + pub(crate) fn proof_object( + &self, + py: Python<'_>, + classes: &CompiledViewClasses<'_>, + ) -> PyResult> { + self.projections + .proof + .get_or_try_init(py, || match &self.proof { + None => Ok(py.None()), + Some(proof) => contract_proof(classes, proof), + }) + .map(|proof| proof.clone_ref(py)) + } +} + +/// `compact_lineage`: rows over a per-model string pool, sources pooled before each output. +fn compact_lineage(classes: &CompiledViewClasses<'_>, rows: &[LineageRow]) -> PyResult> { + let py: Python<'_> = classes.compact_lineage_facts.py(); + let mut pool: StringPool<'_> = StringPool::default(); + let mut compact: Vec> = Vec::with_capacity(rows.len()); + for row in rows { + let mut sources: Vec<(usize, usize, usize)> = Vec::with_capacity(row.sources.len()); + for (resource_type, resource_name, column_name) in &row.sources { + let kind: usize = pool.index(resource_type); + let name: usize = pool.index(resource_name); + sources.push((kind, name, pool.index(column_name))); + } + let output: usize = pool.index(&row.output_column); + compact.push( + ( + output, + row.transform_code, + row.confidence_code, + PyTuple::new(py, sources)?, + ) + .into_pyobject(py)?, + ); + } + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("string_pool", PyTuple::new(py, pool.ordered)?)?; + fields.set_item("rows", PyTuple::new(py, compact)?)?; + Ok(classes + .compact_lineage_facts + .call((), Some(&fields))? + .unbind()) +} + +/// Python's `pool.setdefault(text, len(pool))`, remembering first-seen order. +#[derive(Default)] +struct StringPool<'a> { + indexes: HashMap<&'a str, usize>, + ordered: Vec<&'a str>, +} + +impl<'a> StringPool<'a> { + fn index(&mut self, text: &'a str) -> usize { + if let Some(index) = self.indexes.get(text) { + return *index; + } + self.ordered.push(text); + self.indexes.insert(text, self.ordered.len() - 1); + self.ordered.len() - 1 + } +} + +/// `lineage_facts`: plain `CompiledLineageColumnFact`s of a re-analysis. +fn lineage_facts(classes: &CompiledViewClasses<'_>, rows: &[LineageRow]) -> PyResult> { + let py: Python<'_> = classes.lineage_column_fact.py(); + let mut facts: Vec> = Vec::with_capacity(rows.len()); + for row in rows { + let mut sources: Vec> = Vec::with_capacity(row.sources.len()); + for (resource_type, resource_name, column_name) in &row.sources { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("resource_type", resource_type)?; + fields.set_item("resource_name", resource_name)?; + fields.set_item("column_name", column_name)?; + sources.push( + classes + .lineage_source_fact + .call((), Some(&fields))? + .unbind(), + ); + } + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("output_column", &row.output_column)?; + fields.set_item("upstream_columns", PyTuple::new(py, sources)?)?; + fields.set_item( + "transform_kind", + classes + .compact_lineage_facts + .call_method1("transform_kind", (row.transform_code,))?, + )?; + fields.set_item( + "confidence", + classes + .compact_lineage_facts + .call_method1("confidence", (row.confidence_code,))?, + )?; + facts.push( + classes + .lineage_column_fact + .call((), Some(&fields))? + .unbind(), + ); + } + Ok(PyTuple::new(py, facts)?.into_any().unbind()) +} + +/// One located binding diagnostic as the model's `CompilerDiagnostic`. +fn compiler_diagnostic( + classes: &CompiledViewClasses<'_>, + diagnostic: &LocatedDiagnostic, + (name, path): (&str, &Bound<'_, PyAny>), +) -> PyResult> { + let py: Python<'_> = path.py(); + let location: Option> = match &diagnostic.location { + None => None, + Some(span) => { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("path", path)?; + fields.set_item("line", span.line)?; + fields.set_item("column", span.column)?; + fields.set_item("end_line", span.end_line)?; + fields.set_item("end_column", span.end_column)?; + Some(classes.source_location.call((), Some(&fields))?) + } + }; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("phase", COMPILE_PHASE)?; + fields.set_item("severity", &diagnostic.severity)?; + fields.set_item("code", &diagnostic.code)?; + fields.set_item("message", &diagnostic.message)?; + fields.set_item("resource_type", MODEL_RESOURCE)?; + fields.set_item("resource_name", name)?; + fields.set_item("path", path)?; + fields.set_item("line", diagnostic.line)?; + fields.set_item("column", diagnostic.column)?; + fields.set_item("location", location)?; + Ok(classes + .compiler_diagnostic + .call((), Some(&fields))? + .unbind()) +} + +/// `contract_proof`: the session's dynamic pivot proof as Python's. +fn contract_proof(classes: &CompiledViewClasses<'_>, proof: &ContractProof) -> PyResult> { + let py: Python<'_> = classes.contract_proof.py(); + let mut fixed: Vec> = Vec::with_capacity(proof.fixed_columns.len()); + for column in &proof.fixed_columns { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("name", &column.name)?; + fields.set_item("type", &column.data_type)?; + fields.set_item( + "nullability", + classes.nullability.call1((&column.nullability,))?, + )?; + fixed.push(classes.inferred_column.call((), Some(&fields))?.unbind()); + } + let mut families: Vec> = Vec::with_capacity(proof.families.len()); + for (name, inferred_type) in &proof.families { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("name", name)?; + fields.set_item("inferred_type", inferred_type)?; + families.push(classes.family_proof.call((), Some(&fields))?.unbind()); + } + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("output_proven", proof.output_proven)?; + fields.set_item("fixed_columns", PyTuple::new(py, fixed)?)?; + fields.set_item("families", PyTuple::new(py, families)?)?; + fields.set_item("input_relations", PyTuple::new(py, &proof.input_relations)?)?; + fields.set_item("failure_reason", &proof.failure_reason)?; + fields.set_item("bare_dynamic_pivot", proof.bare_dynamic_pivot)?; + Ok(classes.contract_proof.call((), Some(&fields))?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs new file mode 100644 index 0000000000..b6431540af --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -0,0 +1,734 @@ +//! A compiled model, read from the native compiled project that owns it. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; +use pyo3::{FromPyObject, pyclass, pymethods}; + +use sqlbuild_analysis::compiled_project::models::{DynamicProofFact, ModelAnalysisFact}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_core::text::models::PythonText; + +use crate::bindings::_helpers::compiled_project::compiled_model_analysis::{ + AnalysisRecord, ProofArgument, SessionArgument, analysis_record, +}; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + InputViewClasses, ModelInputRecord, model_relation_key_row, read_input, references, + rejected_opt_out, schema_entry, view_classes, +}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::graph::project_graph::ResourceRow; +use crate::bindings::_helpers::rules::relation_keys::RelationKeyRow; + +const DATABASE_KEY: &str = "database"; +const SCHEMA_KEY: &str = "schema"; +const ALIAS_KEY: &str = "alias"; +const VALUES_ATTRIBUTE: &str = "values"; +const LOGICAL_SCHEMA_ATTRIBUTE: &str = "logical_schema"; +const LOGICAL_DATABASE_ATTRIBUTE: &str = "logical_database"; +const MODEL_RESOURCE: &str = "model"; +const TAGS_KEY: &str = "tags"; +const NULLABLE: &str = "nullable"; + +/// `(name, type, nullability)` of one inferred output column. +pub(crate) type InferredRow = (String, Option, String); + +/// `(name, type)` of a declared column. +pub(crate) type DeclaredRow = (String, Option); + +/// `(database, schema, name)` of a relation. +pub(crate) type DestinationRow = (Option, Option, String); + +/// The Python classes a compiled model's projected attributes are built with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct CompiledViewClasses<'py> { + object_key: Bound<'py, PyAny>, + relation_location: Bound<'py, PyAny>, + pub(crate) inferred_column: Bound<'py, PyAny>, + pub(crate) nullability: Bound<'py, PyAny>, + pub(crate) compiler_diagnostic: Bound<'py, PyAny>, + pub(crate) source_location: Bound<'py, PyAny>, + pub(crate) compact_lineage_facts: Bound<'py, PyAny>, + pub(crate) lineage_column_fact: Bound<'py, PyAny>, + pub(crate) lineage_source_fact: Bound<'py, PyAny>, + pub(crate) contract_proof: Bound<'py, PyAny>, + pub(crate) family_proof: Bound<'py, PyAny>, +} + +/// A compiled relation location's parts. +#[derive(Clone)] +pub(crate) struct DestinationFact { + pub(crate) database: Option, + pub(crate) schema: Option, + pub(crate) name: String, + pub(crate) qualified_name: Option, + pub(crate) logical_schema: Option, + pub(crate) logical_database: Option, +} + +/// What assembly and the semantic checks finished for one compiled model. +pub(crate) struct CompiledRecord { + /// `(resource type, name)` of each dependency. + deps: Vec<(String, String)>, + destination: DestinationFact, + /// Columns, lineage, binding diagnostics and dynamic proof kept from the native session. + analysis: AnalysisRecord, + macro_deps: Vec, + projections: CompiledProjections, +} + +/// A compiled model's Python attribute projections, each built once on first access. +struct CompiledProjections { + key: PyOnceLock>, + deps: PyOnceLock>, +} + +impl Default for CompiledProjections { + fn default() -> Self { + Self { + key: PyOnceLock::new(), + deps: PyOnceLock::new(), + } + } +} + +/// What later stages changed for one view of a model; the owner's record stays as assembled. +#[derive(Default)] +struct ModelOverlay { + binding_diagnostics: Option>, + /// Output columns whose types semantic recovery forgot, left unchecked by the contract. + poisoned: Vec, + destination: Option, +} + +impl ModelOverlay { + fn clone_ref(&self, py: Python<'_>) -> Self { + Self { + binding_diagnostics: self + .binding_diagnostics + .as_ref() + .map(|diagnostics| diagnostics.clone_ref(py)), + poisoned: self.poisoned.clone(), + destination: self.destination.clone(), + } + } +} + +/// One view's projections of attributes its overlay can change, each built once. +struct ViewProjections { + destination: PyOnceLock>, + inferred_columns: PyOnceLock>, +} + +impl Default for ViewProjections { + fn default() -> Self { + Self { + destination: PyOnceLock::new(), + inferred_columns: PyOnceLock::new(), + } + } +} + +/// Deps, session analysis, failed analysis's columns, standalone proof, and macro deps. +#[derive(FromPyObject)] +pub(crate) struct CompiledArguments( + Vec<(String, String)>, + Option, + Option>, + Option, + Vec, +); + +/// The compiled record of one model, with its configured destination. +pub(crate) fn compiled_record( + py: Python<'_>, + input: &ModelInputRecord, + name: &str, + arguments: CompiledArguments, +) -> PyResult { + let CompiledArguments(deps, session, fallback, standalone_proof, macro_deps) = arguments; + let analysis: AnalysisRecord = + analysis_record(py, input, session, (fallback, standalone_proof))?; + let config: &Bound<'_, PyAny> = input.config.bind(py); + let values: Bound<'_, PyDict> = config.getattr(VALUES_ATTRIBUTE)?.downcast_into()?; + Ok(CompiledRecord { + deps, + destination: DestinationFact { + database: text_value(&values, DATABASE_KEY)?, + schema: text_value(&values, SCHEMA_KEY)?, + name: text_value(&values, ALIAS_KEY)?.unwrap_or_else(|| name.to_owned()), + qualified_name: None, + logical_schema: config.getattr(LOGICAL_SCHEMA_ATTRIBUTE)?.extract()?, + logical_database: config.getattr(LOGICAL_DATABASE_ATTRIBUTE)?.extract()?, + }, + analysis, + macro_deps, + projections: CompiledProjections::default(), + }) +} + +fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match values.get_item(key)? { + Some(value) if value.is_instance_of::() => { + Ok(Some(value.extract()?)) + } + _ => Ok(None), + } +} + +/// The destination with adapter defaults filled in; `render` renders its qualified name. +pub(crate) fn defaulted_destination( + current: &DestinationFact, + defaults: (Option, Option), + render: &Bound<'_, PyAny>, +) -> PyResult { + let (default_schema, default_database) = defaults; + let schema: Option = current.schema.clone().or(default_schema); + let database: Option = current.database.clone().or(default_database); + let keywords: Bound<'_, PyDict> = PyDict::new(render.py()); + keywords.set_item(DATABASE_KEY, &database)?; + keywords.set_item(SCHEMA_KEY, &schema)?; + keywords.set_item("name", ¤t.name)?; + let qualified_name: Option = render.call((), Some(&keywords))?.extract()?; + Ok(DestinationFact { + database, + schema, + name: current.name.clone(), + qualified_name, + logical_schema: current.logical_schema.clone(), + logical_database: current.logical_database.clone(), + }) +} + +/// One compiled model, projected from the owner on demand. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeCompiledModel { + owner: Py, + index: usize, + overlay: ModelOverlay, + projections: ViewProjections, +} + +impl NativeCompiledModel { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self::with_overlay(owner, index, ModelOverlay::default()) + } + + fn with_overlay(owner: Py, index: usize, overlay: ModelOverlay) -> Self { + Self { + owner, + index, + overlay, + projections: ViewProjections::default(), + } + } + + fn changed(&self, py: Python<'_>, change: impl FnOnce(&mut ModelOverlay)) -> Self { + let mut overlay: ModelOverlay = self.overlay.clone_ref(py); + change(&mut overlay); + Self::with_overlay(self.owner.clone_ref(py), self.index, overlay) + } + + fn read<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, + ) -> PyResult { + read_input(&self.owner, self.index, py, read) + } + + fn compiled<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&CompiledRecord, Python<'py>) -> PyResult, + ) -> PyResult { + self.read(py, |record, py| read(compiled_of(record)?, py)) + } + + /// This view's typed analysis as the owner's model facts read it, and the model's name. + pub(crate) fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { + self.read(py, |record, py| { + let analysis: &AnalysisRecord = &compiled_of(record)?.analysis; + let rows: &[InferredRow] = analysis.inferred.as_deref().unwrap_or_default(); + let python: PythonText = active_python_text(); + let mut inferred_columns: Vec<(String, Option)> = + Vec::with_capacity(rows.len()); + for (name, column_type, _) in rows { + let poisoned: bool = self.overlay.poisoned.contains(name); + inferred_columns.push((name.clone(), column_type.clone().filter(|_| !poisoned))); + } + let dynamic_proof: Option = + analysis.proof.as_ref().map(|proof| DynamicProofFact { + output_proven: proof.output_proven, + bare_dynamic_pivot: proof.bare_dynamic_pivot, + families: casefolded_families(python, &proof.families), + failure_reason: proof.failure_reason.clone(), + named_families: proof.families.clone(), + }); + let fact: ModelAnalysisFact = ModelAnalysisFact { + inferred_columns, + dynamic_proof, + inferred: analysis.inferred.is_some(), + nullable_columns: rows + .iter() + .map(|(_, _, nullability)| nullability == NULLABLE) + .collect(), + fast_lineage_has_star: analysis.has_star, + unchecked_output_columns: self.overlay.poisoned.clone(), + }; + Ok((model_name(record, py)?, fact)) + }) + } + + fn model_file_attribute(&self, py: Python<'_>, name: &str) -> PyResult> { + self.read(py, |record, py| { + Ok(record.model_file.bind(py).getattr(name)?.unbind()) + }) + } +} + +/// Each family's name casefolded as Python's, with its inferred type. +fn casefolded_families( + python: PythonText, + families: &[(String, Option)], +) -> Vec<(String, Option)> { + families + .iter() + .map(|(name, kind)| (python_casefold(python, name), kind.clone())) + .collect() +} + +/// The compiled record of an input, which assembly records before any view is handed out. +fn compiled_of(record: &ModelInputRecord) -> PyResult<&CompiledRecord> { + record + .compiled + .as_ref() + .ok_or_else(|| pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled")) +} + +#[pymethods] +impl NativeCompiledModel { + #[getter] + fn key(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + let compiled: &CompiledRecord = compiled_of(record)?; + compiled + .projections + .key + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + object_key(&classes.compiled, MODEL_RESOURCE, &model_name(record, py)?) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + fn deps(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, py| { + compiled + .projections + .deps + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let mut keys: Vec> = Vec::with_capacity(compiled.deps.len()); + for (kind, name) in &compiled.deps { + keys.push(object_key(&classes.compiled, kind, name)?); + } + Ok::<_, pyo3::PyErr>(PyTuple::new(py, keys)?.unbind()) + }) + .map(|deps| deps.clone_ref(py)) + }) + } + + #[getter] + fn name(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, py| model_name(record, py)) + } + + #[getter] + fn relative_path(&self, py: Python<'_>) -> PyResult> { + self.model_file_attribute(py, "relative_path") + } + + #[getter] + fn query_sql(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| Ok(record.query_sql.clone_ref(py))) + } + + #[getter] + fn config(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.config.clone_ref(py))) + } + + #[getter] + fn destination(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, py| { + self.projections + .destination + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let destination: &DestinationFact = self + .overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination); + relation_location(&classes.compiled, destination) + }) + .map(|destination| destination.clone_ref(py)) + }) + } + + #[getter] + fn references(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| references(&classes, record)) + } + + /// Each reference as `(kind, name, package, call argument count)`. + #[getter] + fn reference_rows(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| Ok(record.references.clone())) + } + + /// The model's declared key facts, for deriving its unique keys. + #[getter] + fn relation_key_row(&self, py: Python<'_>) -> PyResult { + self.read(py, model_relation_key_row) + } + + /// `(resource type, name)` of the model's key. + #[getter] + fn key_row(&self, py: Python<'_>) -> PyResult<(String, String)> { + self.read(py, |record, py| { + Ok((MODEL_RESOURCE.to_owned(), model_name(record, py)?)) + }) + } + + /// `(resource type, name)` of each dependency. + #[getter] + fn dep_rows(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, _| Ok(compiled.deps.clone())) + } + + /// `(database, schema, name)` of the model's destination. + #[getter] + fn destination_row(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| { + let destination: &DestinationFact = self + .overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination); + Ok(( + destination.database.clone(), + destination.schema.clone(), + destination.name.clone(), + )) + }) + } + + /// Each inferred column as `(name, type, nullability)`, or `None` when none were inferred. + #[getter] + fn inferred_rows(&self, py: Python<'_>) -> PyResult>> { + self.compiled(py, |compiled, _| { + let Some(rows) = &compiled.analysis.inferred else { + return Ok(None); + }; + let mut shown: Vec = Vec::with_capacity(rows.len()); + for (name, column_type, nullability) in rows { + let poisoned: bool = self.overlay.poisoned.contains(name); + let column_type: Option = if poisoned { None } else { column_type.clone() }; + shown.push((name.clone(), column_type, nullability.clone())); + } + Ok(Some(shown)) + }) + } + + /// Each declared column of the attached schema as `(name, type)`, or `None` without one. + #[getter] + fn schema_column_rows(&self, py: Python<'_>) -> PyResult>> { + self.read(py, |record, py| { + let Some(schema) = &record.schema else { + return Ok(None); + }; + let mut rows: Vec = Vec::new(); + for column in schema.columns.bind(py).iter() { + rows.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + Ok(Some(rows)) + }) + } + + /// Whether the attached schema declares dynamic columns. + #[getter(has_dynamic_columns)] + fn dynamic_columns_declared(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, py| { + Ok(record + .schema + .as_ref() + .is_some_and(|schema| !schema.dynamic_columns.bind(py).is_empty())) + }) + } + + /// Inferred columns, else the attached schema's declared columns, else zero. + #[getter] + fn output_column_count(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, py| { + if let Some(rows) = &compiled_of(record)?.analysis.inferred { + return Ok(rows.len()); + } + Ok(record + .schema + .as_ref() + .map_or(0, |schema| schema.columns.bind(py).len())) + }) + } + + #[getter] + fn schema_entry(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| schema_entry(&classes, record)) + } + + #[getter] + fn inferred_columns(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, py| { + self.projections + .inferred_columns + .get_or_try_init(py, || { + let Some(rows) = &compiled.analysis.inferred else { + return Ok::<_, pyo3::PyErr>(py.None()); + }; + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let mut columns: Vec> = Vec::with_capacity(rows.len()); + for row in rows { + let poisoned_row: InferredRow; + let row = if self.overlay.poisoned.contains(&row.0) { + poisoned_row = (row.0.clone(), None, row.2.clone()); + &poisoned_row + } else { + row + }; + columns.push(self.owner.bind(py).borrow().inferred_column( + py, + &classes.compiled, + row, + )?); + } + Ok(PyTuple::new(py, columns)?.into_any().unbind()) + }) + .map(|columns| columns.clone_ref(py)) + }) + } + + #[getter] + fn fast_lineage_columns(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.compiled(py, |compiled, py| { + compiled.analysis.lineage_object(py, &classes.compiled) + }) + } + + #[getter] + fn fast_lineage_has_star(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| Ok(compiled.analysis.has_star)) + } + + #[getter] + fn fast_lineage_star_resolved(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| Ok(compiled.analysis.star_resolved)) + } + + #[getter] + fn authored_sql(&self, py: Python<'_>) -> PyResult> { + self.model_file_attribute(py, "contents") + } + + #[getter] + fn authored_query_sql(&self, py: Python<'_>) -> PyResult> { + self.model_file_attribute(py, "query_sql") + } + + #[getter] + fn output_column_locations(&self, py: Python<'_>) -> PyResult> { + self.model_file_attribute(py, "output_column_locations") + } + + #[getter] + fn extract_implicit_alias_columns(&self, py: Python<'_>) -> PyResult> { + self.model_file_attribute(py, "extract_implicit_alias_columns") + } + + #[getter] + fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { + self.compiled(py, |compiled, py| PyTuple::new(py, &compiled.macro_deps)) + } + + #[getter] + fn enum_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.enum_declarations.clone_ref(py))) + } + + #[getter] + fn constant_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.constant_declarations.clone_ref(py)) + }) + } + + #[getter] + fn enum_columns(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.enum_columns.clone_ref(py))) + } + + #[getter] + fn binding_diagnostics(&self, py: Python<'_>) -> PyResult> { + if let Some(diagnostics) = &self.overlay.binding_diagnostics { + return Ok(diagnostics.clone_ref(py)); + } + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, py| { + let name: String = model_name(record, py)?; + let path: Bound<'_, PyAny> = record.model_file.bind(py).getattr("relative_path")?; + compiled_of(record)? + .analysis + .binding_tuple(py, &classes.compiled, (&name, &path)) + }) + } + + #[getter] + fn binding_validated(&self, py: Python<'_>) -> PyResult { + self.compiled(py, |compiled, _| Ok(compiled.analysis.binding_validated)) + } + + #[getter] + fn dynamic_column_contract(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.compiled(py, |compiled, py| { + compiled.analysis.proof_object(py, &classes.compiled) + }) + } + + #[getter] + fn unchecked_output_columns<'py>(&self, py: Python<'py>) -> PyResult> { + PyFrozenSet::new(py, &self.overlay.poisoned) + } + + #[getter] + fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(rejected_opt_out(record, py))) + } + + /// This model with its final binding diagnostics. + fn with_binding_diagnostics(&self, py: Python<'_>, diagnostics: Py) -> Self { + self.changed(py, |overlay| { + overlay.binding_diagnostics = Some(diagnostics) + }) + } + + /// This model after semantic type recovery: its bindings and the outputs left unchecked. + fn with_type_recovery( + &self, + py: Python<'_>, + diagnostics: Py, + poisoned: Vec, + ) -> Self { + self.changed(py, |overlay| { + overlay.binding_diagnostics = Some(diagnostics); + overlay.poisoned = poisoned; + }) + } + + /// This model with the adapter's default schema and database filled into its destination. + fn with_target_defaults( + &self, + py: Python<'_>, + defaults: (Option, Option), + render: Bound<'_, PyAny>, + ) -> PyResult { + let destination: DestinationFact = self.compiled(py, |compiled, _| { + let current: &DestinationFact = self + .overlay + .destination + .as_ref() + .unwrap_or(&compiled.destination); + defaulted_destination(current, defaults, &render) + })?; + Ok(self.changed(py, |overlay| overlay.destination = Some(destination))) + } +} + +/// The model's project graph row: its key, dependencies, tags and folder. +pub(crate) fn model_graph_row( + model: &NativeCompiledModel, + py: Python<'_>, +) -> PyResult { + model.read(py, |record, py| { + let compiled: &CompiledRecord = compiled_of(record)?; + let values: Bound<'_, PyAny> = record.config.bind(py).getattr(VALUES_ATTRIBUTE)?; + let mut tags: Vec = Vec::new(); + if let Some(value) = values.downcast::()?.get_item(TAGS_KEY)? + && (value.is_instance_of::() + || value.is_instance_of::()) + { + for tag in value.try_iter()? { + tags.push(tag?.str()?.extract()?); + } + } + let folder: String = record + .model_file + .bind(py) + .getattr("relative_path")? + .getattr("parent")? + .str()? + .extract()?; + Ok(( + (MODEL_RESOURCE.to_owned(), model_name(record, py)?), + compiled.deps.clone(), + tags, + Some(folder), + )) + }) +} + +fn model_name(record: &ModelInputRecord, py: Python<'_>) -> PyResult { + record + .model_file + .bind(py) + .getattr("file_path")? + .getattr("stem")? + .extract() +} + +/// The `CompiledRelationLocation` of a destination. +pub(crate) fn relation_location( + classes: &CompiledViewClasses<'_>, + destination: &DestinationFact, +) -> PyResult> { + let fields: Bound<'_, PyDict> = PyDict::new(classes.relation_location.py()); + fields.set_item(DATABASE_KEY, &destination.database)?; + fields.set_item(SCHEMA_KEY, &destination.schema)?; + fields.set_item("name", &destination.name)?; + fields.set_item("qualified_name", &destination.qualified_name)?; + fields.set_item(LOGICAL_SCHEMA_ATTRIBUTE, &destination.logical_schema)?; + fields.set_item(LOGICAL_DATABASE_ATTRIBUTE, &destination.logical_database)?; + Ok(classes.relation_location.call((), Some(&fields))?.unbind()) +} + +pub(crate) fn object_key( + classes: &CompiledViewClasses<'_>, + kind: &str, + name: &str, +) -> PyResult> { + let fields: Bound<'_, PyDict> = PyDict::new(classes.object_key.py()); + fields.set_item("resource_type", kind)?; + fields.set_item("name", name)?; + Ok(classes.object_key.call((), Some(&fields))?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs new file mode 100644 index 0000000000..7b5465fe3e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs @@ -0,0 +1,590 @@ +//! `build_sql_function_inputs`: every SQL and Python function attached natively. + +use std::collections::{HashMap, HashSet}; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use sqlbuild_analysis::assembly::analysis_session::main::query_columns::query_columns; +use sqlbuild_analysis::assembly::analysis_session::models::{ + QueryColumns, QueryColumnsInput, QueryColumnsMode, QueryColumnsRequest, +}; +use sqlbuild_analysis::assembly::project::errors::AssemblyError; +use sqlbuild_analysis::assembly::project::main::preserved_logical_namespace::preserved_logical_namespace; +use sqlbuild_analysis::assembly::project::models::TargetNamespace; +use sqlbuild_analysis::compiled_project::main::located_scan_error::located_scan_error; +use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge_call_site_references; +use sqlbuild_analysis::compiled_project::models::ScanFaultOrigin; +use sqlbuild_analysis::compiler::main::function_type_validation::function_type_error; +use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; +use sqlbuild_attachments::functions::main::function_reference_error::function_reference_error; +use sqlbuild_attachments::functions::main::resolve_function_namespace::resolve_function_namespace; +use sqlbuild_attachments::functions::main::table_function_arity_error::table_function_arity_error; +use sqlbuild_attachments::functions::models::{ + FunctionHeader, FunctionLanguage, FunctionNamespace, FunctionReturns, HeaderStage, + KnownResources, NamedType, NamespaceInputs, +}; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; + +use crate::bindings::_helpers::analysis_session::query_columns::query_columns_error; +use crate::bindings::_helpers::attachments::function_headers::parsed_header; +use crate::bindings::_helpers::compiled_project::compiled_model_view::DestinationFact; +use crate::bindings::_helpers::compiled_project::function_views::{ + AttachedFunction, NamedTypeRow, PythonRuntime, +}; +use crate::bindings::_helpers::compiled_project::model_loop::{ + LoopContext, NativeModelLoop, compile_error, +}; +use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::model_loop_inputs::reference_row; +use crate::bindings::_helpers::compiled_project::source_loop::{ + AuthoredExpansion, expanded_authored_sql, expanded_template, parse_checked, scoped_declarations, +}; +use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; + +const PRESERVE_TARGET_VALUE: &str = "preserve"; +const GENERIC_DIALECT: &str = "generic"; +const TABLE_RETURNS: &str = "TABLE"; +const DATABASE_KEY: &str = "database"; +const SCHEMA_KEY: &str = "schema"; + +/// A function's paths, stem, discovered file, header values, body and SQL file contents. +pub(crate) type FunctionRow = ( + String, + String, + String, + Py, + Py, + String, + Option, +); + +/// Adapter, validation, raw default and target namespaces, inheritance and resource kind. +pub(crate) type FunctionSettings = ( + String, + bool, + (Option, Option), + Option<(Option, Option)>, + bool, + Py, +); + +/// What every function in one compile resolves against. +struct FunctionScope<'a, 'py> { + model_loop: &'a NativeModelLoop, + context: &'a LoopContext<'a, 'py>, + adapter_name: String, + validate: bool, + defaults: (Option, Option), + targets: (Option, Option), + target: Option, + inherit: bool, + function_kind: Bound<'py, PyAny>, +} + +/// Attach every SQL function, then every Python function, in discovery order. +pub(crate) fn attached_functions<'py>( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, 'py>, + rows: (Vec, Vec), + settings: FunctionSettings, +) -> PyResult> { + let py: Python<'py> = context.host.py(); + let (adapter_name, validate, (default_database, default_schema), target, inherit, kind) = + settings; + let defaults = ( + optional_template(context, default_database, "default function database")?, + optional_template(context, default_schema, "default function schema")?, + ); + let targets = match &target { + Some((database, schema)) => ( + environment_value(context, database.as_deref(), "target database")?, + environment_value(context, schema.as_deref(), "target schema")?, + ), + None => (None, None), + }; + let scope = FunctionScope { + model_loop, + context, + adapter_name, + validate, + defaults, + targets, + target: target.map(|(database, schema)| TargetNamespace { + database, + schema, + loader_schema: None, + }), + inherit, + function_kind: kind.into_bound(py), + }; + let (sql_rows, python_rows) = rows; + let mut names: HashSet = HashSet::new(); + let mut attached: Vec = + Vec::with_capacity(sql_rows.len() + python_rows.len()); + for row in sql_rows { + if !names.insert(row.2.clone()) { + let message: String = format!("Duplicate SQL function name '{}'", row.2); + return Err(compile_error(context.classes, &message, None)); + } + attached.push(sql_function(&scope, row)?); + } + for row in python_rows { + if !names.insert(row.2.clone()) { + let message: String = format!("Duplicate function name '{}'", row.2); + return Err(compile_error(context.classes, &message, None)); + } + attached.push(python_function(&scope, row)?); + } + Ok(attached) +} + +/// One SQL function, raising each header error where Python's attachment raises it. +fn sql_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult { + let context: &LoopContext<'_, '_> = scope.context; + let py: Python<'_> = context.host.py(); + let (file_path, relative_path, name, function_file, header_values, body, contents) = row; + let header_values: Bound<'_, PyDict> = header_values.into_bound(py); + let label: String = format!("SQL function {relative_path}"); + let header: FunctionHeader = + parsed_header(&header_values, FunctionLanguage::Sql, &relative_path); + raise_stage(context, &header, HeaderStage::Start)?; + let arguments: Vec = expanded_types(context, &header.arguments, |raw| { + format!("{label} argument '{raw}' type") + })?; + raise_stage(context, &header, HeaderStage::Arguments)?; + let (returns, return_columns): (String, Vec) = match &header.returns { + Some(FunctionReturns::Table(columns)) => ( + TABLE_RETURNS.to_owned(), + expanded_types(context, columns, |raw| { + format!("{label} return column '{raw}' type") + })?, + ), + Some(FunctionReturns::Type(text)) => ( + header_template(context, text, &format!("{label} returns"))?, + Vec::new(), + ), + None => (String::from("None"), Vec::new()), + }; + raise_stage(context, &header, HeaderStage::Returns)?; + let namespace: FunctionNamespace = + function_namespace(scope, &header_values, &label, FunctionLanguage::Sql)?; + preserved( + scope, + &format!("SQL function '{name}'"), + (&namespace.logical_database, &namespace.logical_schema), + )?; + let consumer: Bound<'_, PyAny> = context + .classes + .visibility + .resource_identity + .call1((&scope.function_kind, &name))?; + let declarations: Bound<'_, PyAny> = scoped_declarations(context, &file_path, &consumer)?; + let expansion: AuthoredExpansion<'_> = expanded_authored_sql( + scope.model_loop, + context, + (&body, &file_path), + &declarations, + )?; + let sql: &str = &expansion.sql; + if let IntrinsicCheck::Rejected(message) = intrinsic_free( + scope.model_loop.python, + sql, + &scope.model_loop.reserved_markers, + &format!("SQL function '{name}'"), + ) { + return Err(compile_error(context.classes, &message, None)); + } + if scope.validate { + validated_types(scope, &label, &arguments, (&returns, &return_columns))?; + parse_checked( + context, + sql, + &format!("SQL syntax error in function '{name}' ({file_path})"), + )?; + if !return_columns.is_empty() { + checked_output_contract(sql, return_columns.len(), &relative_path, context)?; + } + } + let contents: String = contents.unwrap_or_default(); + let references: Vec = function_references( + scope, + (&file_path, &relative_path, &contents), + sql, + expansion.argument_references, + )?; + raise_stage(context, &header, HeaderStage::Metadata)?; + Ok(AttachedFunction { + function_file, + relative_path, + destination: destination(&name, &namespace, false), + fingerprint_destination: destination(&name, &namespace, true), + name, + arguments, + returns, + return_columns, + body: expansion.sql, + references, + python: None, + tags: header.tags, + description: header.description, + usages: expansion.usages.unbind(), + }) +} + +/// One Python function, raising each header error where Python's attachment raises it. +fn python_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult { + let context: &LoopContext<'_, '_> = scope.context; + let py: Python<'_> = context.host.py(); + let (_, relative_path, name, function_file, header_values, body, _) = row; + let header_values: Bound<'_, PyDict> = header_values.into_bound(py); + let label: String = format!("Python function {relative_path}"); + let header: FunctionHeader = + parsed_header(&header_values, FunctionLanguage::Python, &relative_path); + raise_stage(context, &header, HeaderStage::Start)?; + let arguments: Vec = expanded_types(context, &header.arguments, |raw| { + format!("{label} argument '{raw}' type") + })?; + raise_stage(context, &header, HeaderStage::Arguments)?; + let returns: String = match &header.returns { + Some(FunctionReturns::Type(text)) => { + header_template(context, text, &format!("{label} returns"))? + } + _ => String::from("None"), + }; + raise_stage(context, &header, HeaderStage::PythonValues)?; + let namespace: FunctionNamespace = + function_namespace(scope, &header_values, &label, FunctionLanguage::Python)?; + preserved( + scope, + &format!("Python function '{name}'"), + ( + &namespace.fingerprint_logical_database, + &namespace.fingerprint_logical_schema, + ), + )?; + if scope.validate { + validated_types(scope, &label, &arguments, (&returns, &[]))?; + } + raise_stage(context, &header, HeaderStage::Metadata)?; + Ok(AttachedFunction { + function_file, + relative_path, + destination: destination(&name, &namespace, false), + fingerprint_destination: destination(&name, &namespace, true), + name, + arguments, + returns, + return_columns: Vec::new(), + body, + references: Vec::new(), + python: Some(PythonRuntime { + runtime_version: header.runtime_version, + entry_point: header.entry_point, + packages: header.packages, + }), + tags: header.tags, + description: header.description, + usages: PyTuple::empty(py).unbind(), + }) +} + +/// `_raise_header_failure`: the header's error when it is raised at `stage`. +fn raise_stage( + context: &LoopContext<'_, '_>, + header: &FunctionHeader, + stage: HeaderStage, +) -> PyResult<()> { + match &header.failure { + Some(failure) if failure.stage == stage => { + Err(compile_error(context.classes, &failure.message, None)) + } + _ => Ok(()), + } +} + +/// Each `(name, type)` with its type's templates expanded, in order. +fn expanded_types( + context: &LoopContext<'_, '_>, + types: &[NamedType], + label: impl Fn(&str) -> String, +) -> PyResult> { + types + .iter() + .map(|named| { + Ok(( + named.name.clone(), + header_template(context, &named.type_text, &label(&named.raw_name))?, + )) + }) + .collect() +} + +/// `_expand_function_header_value`: templates expanded, `@@CTX` tokens kept. +fn header_template(context: &LoopContext<'_, '_>, raw: &str, label: &str) -> PyResult { + let py: Python<'_> = context.host.py(); + expanded_template(context, PyString::new(py, raw).as_any(), label, true)? + .str()? + .extract() +} + +fn optional_template( + context: &LoopContext<'_, '_>, + raw: Option, + label: &str, +) -> PyResult> { + raw.map(|raw| header_template(context, &raw, label)) + .transpose() +} + +/// `_expand_function_environment_value`: a preserved target dimension is unset. +fn environment_value( + context: &LoopContext<'_, '_>, + raw: Option<&str>, + label: &str, +) -> PyResult> { + match raw { + None | Some(PRESERVE_TARGET_VALUE) => Ok(None), + Some(raw) => Ok(Some(header_template(context, raw, label)?)), + } +} + +/// The function's namespace from its own string `database` and `schema` header values. +fn function_namespace( + scope: &FunctionScope<'_, '_>, + header_values: &Bound<'_, PyDict>, + label: &str, + language: FunctionLanguage, +) -> PyResult { + let mut header: [Option; 2] = [None, None]; + for (slot, key) in header.iter_mut().zip([DATABASE_KEY, SCHEMA_KEY]) { + if let Some(value) = header_values.get_item(key)? + && let Ok(text) = value.downcast::() + { + let text: String = text.to_string_lossy().into_owned(); + *slot = Some(header_template( + scope.context, + &text, + &format!("{label} {key}"), + )?); + } + } + let [header_database, header_schema] = header; + Ok(resolve_function_namespace(&NamespaceInputs { + header_database, + header_schema, + default_database: scope.defaults.0.clone(), + default_schema: scope.defaults.1.clone(), + target_database: scope.targets.0.clone(), + target_schema: scope.targets.1.clone(), + language, + inherit_default_namespace: scope.inherit, + })) +} + +/// `validate_preserved_logical_namespace`. +fn preserved( + scope: &FunctionScope<'_, '_>, + label: &str, + logical: (&Option, &Option), +) -> PyResult<()> { + match preserved_logical_namespace( + label, + scope.target.as_ref(), + (logical.0.as_ref(), logical.1.as_ref()), + ) { + Ok(()) => Ok(()), + Err(AssemblyError::CompileInput { message, help }) => { + let py: Python<'_> = scope.context.host.py(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", help)?; + let error: Bound<'_, PyAny> = scope + .context + .classes + .compile_error + .call((message,), Some(&keywords))?; + Err(PyErr::from_value(error)) + } + Err(other) => Err(PyValueError::new_err(format!("{other:?}"))), + } +} + +/// `validate_native_type` for every argument, then the return columns or return type. +fn validated_types( + scope: &FunctionScope<'_, '_>, + label: &str, + arguments: &[NamedTypeRow], + returns: (&str, &[NamedTypeRow]), +) -> PyResult<()> { + for (name, type_text) in arguments { + validated_type(scope, type_text, &format!("{label} argument '{name}'"))?; + } + let (returns, return_columns) = returns; + if return_columns.is_empty() { + return validated_type(scope, returns, &format!("{label} return type")); + } + for (name, type_text) in return_columns { + validated_type(scope, type_text, &format!("{label} return column '{name}'"))?; + } + Ok(()) +} + +fn validated_type(scope: &FunctionScope<'_, '_>, type_sql: &str, label: &str) -> PyResult<()> { + match function_type_error(type_sql, &scope.adapter_name, label) { + Ok(None) => Ok(()), + Ok(Some(message)) => Err(compile_error(scope.context.classes, &message, None)), + Err(message) => Err(PyValueError::new_err(message)), + } +} + +/// `_validate_table_function_output_contract`: the body projects as many columns as declared. +fn checked_output_contract( + sql: &str, + declared: usize, + relative_path: &str, + context: &LoopContext<'_, '_>, +) -> PyResult<()> { + let request = QueryColumnsRequest { + dialect: GENERIC_DIALECT.to_owned(), + normalization_dialect: GENERIC_DIALECT.to_owned(), + function_return_types: Vec::new(), + nullability_rules: Some(Vec::new()), + nullability_callback: None, + types: Vec::new(), + nullability: Vec::new(), + queries: vec![QueryColumnsInput { + sql: sql.to_owned(), + placeholders: Vec::new(), + analysis_names: Vec::new(), + lineage_references: Vec::new(), + recover_cte_facts: false, + mode: QueryColumnsMode::Parse, + }], + }; + let columns: Vec = query_columns(None, &request).map_err(query_columns_error)?; + let inferred: usize = columns + .first() + .and_then(|query| query.columns.as_ref()) + .map_or(0, Vec::len); + if inferred == 0 || inferred == declared { + return Ok(()); + } + let message: String = format!( + "SQL table function {relative_path} declares {declared} return columns but its query \ + produces {inferred}" + ); + Err(compile_error(context.classes, &message, None)) +} + +/// The body's references, reporting rejected calls, then validated against the project. +fn function_references( + scope: &FunctionScope<'_, '_>, + file: (&str, &str, &str), + sql: &str, + argument_references: Vec, +) -> PyResult> { + let (file_path, relative_path, contents) = file; + let context: &LoopContext<'_, '_> = scope.context; + let syntax = &scope.model_loop.syntax; + let scanned = match extract_sql_references(sql, syntax) { + ReferenceExtraction::Extracted(scanned) => scanned, + ReferenceExtraction::Failed(failure) => { + let origin: ScanFaultOrigin<'_> = ScanFaultOrigin { + relative_path, + contents, + query_sql: sql, + passes: None, + }; + let message: String = located_scan_error(&failure, sql, &origin, syntax); + return Err(compile_error(context.classes, &message, None)); + } + }; + report_invalid_calls( + context, + syntax, + &scanned.invalid_calls, + (None, relative_path, file_path, contents), + )?; + let references: Vec = merge_call_site_references( + scanned.references.iter().map(reference_row).collect(), + argument_references, + ); + let validator: Bound<'_, NativeModelValidator> = scope + .model_loop + .validator + .bind(context.host.py()) + .downcast()? + .clone(); + let known: KnownResources<'_> = validator.get().known_resources(); + let calls: Vec<(&str, &str)> = references + .iter() + .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) + .collect(); + if let Some(message) = function_reference_error(relative_path, &calls, &known) { + return Err(compile_error(context.classes, &message, None)); + } + Ok(references) +} + +/// The physical, or fingerprint, destination of one function. +fn destination(name: &str, namespace: &FunctionNamespace, fingerprint: bool) -> DestinationFact { + let (database, schema, logical_database, logical_schema) = if fingerprint { + ( + &namespace.fingerprint_database, + &namespace.fingerprint_schema, + &namespace.fingerprint_logical_database, + &namespace.fingerprint_logical_schema, + ) + } else { + ( + &namespace.database, + &namespace.schema, + &namespace.logical_database, + &namespace.logical_schema, + ) + }; + DestinationFact { + database: database.clone(), + schema: schema.clone(), + name: name.to_owned(), + qualified_name: None, + logical_schema: logical_schema.clone(), + logical_database: logical_database.clone(), + } +} + +/// `validate_table_function_call_arities` over each model's, then each function's, calls. +pub(crate) fn checked_table_function_arities( + context: &LoopContext<'_, '_>, + models: &[(String, Vec)], + functions: &[AttachedFunction], +) -> PyResult<()> { + let counts: HashMap = functions + .iter() + .filter(|function| !function.return_columns.is_empty()) + .map(|function| (function.name.clone(), function.arguments.len())) + .collect(); + let mut owners: Vec<(String, &[ReferenceRow])> = Vec::new(); + for (path, references) in models { + owners.push((format!("Model file {path}"), references)); + } + for function in functions { + owners.push(( + format!("SQL function file {}", function.relative_path), + &function.references, + )); + } + for (owner, references) in owners { + let calls: Vec<(&str, &str, Option)> = references + .iter() + .map(|(kind, name, _, count)| (kind.as_str(), name.as_str(), *count)) + .collect(); + if let Some(message) = table_function_arity_error(&owner, &calls, &counts) { + return Err(compile_error(context.classes, &message, None)); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs new file mode 100644 index 0000000000..2c75f75807 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs @@ -0,0 +1,564 @@ +//! Functions attached by the native loop, held by the compiled project, and their views. + +use pyo3::exceptions::{PyIndexError, PyRuntimeError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{FromPyObject, pyclass, pymethods}; + +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + DestinationFact, defaulted_destination, object_key, relation_location, +}; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + InputViewClasses, reference_tuple, view_classes, +}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; + +const TABLE_FUNCTION_TYPE: &str = "table_fn"; +const SCALAR_FUNCTION_TYPE: &str = "udf"; +const SQL_LANGUAGE: &str = "sql"; +const PYTHON_LANGUAGE: &str = "python"; +const FUNCTION_FILE_ATTRIBUTE: &str = "relative_path"; +const FILE_PATH_ATTRIBUTE: &str = "file_path"; +const NOT_ASSEMBLED: &str = "the function was not assembled"; +const FUNCTION_CLASSES: &str = "functions"; + +/// The Python classes a function view projects its argument and return column records with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct FunctionClasses<'py> { + function_argument: Bound<'py, PyAny>, + function_return_column: Bound<'py, PyAny>, + function_language: Bound<'py, PyAny>, +} + +/// `(name, type)` of an argument or return column, its types template-expanded. +pub(crate) type NamedTypeRow = (String, String); + +/// `(node type, name, tags, references as (kind, name, package))`. +pub(crate) type FunctionAssemblyRow = ( + &'static str, + String, + Vec, + Vec<(String, String, Option)>, +); + +/// `(kind, name)` of a compiled object key. +pub(crate) type ObjectKeyRow = (String, String); + +/// A Python function's runtime, entry point and packages. +pub(crate) struct PythonRuntime { + pub(crate) runtime_version: Option, + pub(crate) entry_point: Option, + pub(crate) packages: Vec, +} + +/// What the function loop attached for one SQL or Python function. +pub(crate) struct AttachedFunction { + /// The discovered function file. + pub(crate) function_file: Py, + /// The file's relative path, as messages name it. + pub(crate) relative_path: String, + pub(crate) name: String, + pub(crate) arguments: Vec, + pub(crate) returns: String, + pub(crate) return_columns: Vec, + /// The expanded SQL body, or the Python function's source. + pub(crate) body: String, + pub(crate) references: Vec, + /// The physical and logical destination. + pub(crate) destination: DestinationFact, + /// The destination the function's fingerprint reads. + pub(crate) fingerprint_destination: DestinationFact, + /// `None` for a SQL function. + pub(crate) python: Option, + pub(crate) tags: Vec, + pub(crate) description: Option, + /// Usage records, the identities the scope index is built from. + pub(crate) usages: Py, +} + +impl AttachedFunction { + /// `function_node_type`: a table function when it declares return columns. + pub(crate) fn node_type(&self) -> &'static str { + if self.return_columns.is_empty() { + SCALAR_FUNCTION_TYPE + } else { + TABLE_FUNCTION_TYPE + } + } +} + +/// One attached function held by the compiled project. +pub(crate) struct FunctionRecord { + pub(crate) attached: AttachedFunction, + /// The dependency keys native project assembly resolved, once it ran. + deps: Option>, + arguments: PyOnceLock>, + return_columns: PyOnceLock>, + references: PyOnceLock>, + key: PyOnceLock>, +} + +impl FunctionRecord { + pub(crate) fn new(attached: AttachedFunction) -> Self { + Self { + attached, + deps: None, + arguments: PyOnceLock::new(), + return_columns: PyOnceLock::new(), + references: PyOnceLock::new(), + key: PyOnceLock::new(), + } + } + + /// Keep the dependency keys native project assembly resolved for the function. + pub(crate) fn set_deps(&mut self, deps: Vec) { + self.deps = Some(deps); + } + + fn named_types( + rows: &[NamedTypeRow], + class: &Bound<'_, PyAny>, + cache: &PyOnceLock>, + ) -> PyResult> { + let py: Python<'_> = class.py(); + cache + .get_or_try_init(py, || { + let mut items: Vec> = Vec::with_capacity(rows.len()); + for (name, type_text) in rows { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("name", name)?; + fields.set_item("type", type_text)?; + items.push(class.call((), Some(&fields))?); + } + Ok::<_, pyo3::PyErr>(PyTuple::new(py, items)?.unbind()) + }) + .map(|items| items.clone_ref(py)) + } + + fn language<'py>(&self, classes: &FunctionClasses<'py>) -> PyResult> { + let value: &str = if self.attached.python.is_some() { + PYTHON_LANGUAGE + } else { + SQL_LANGUAGE + }; + classes.function_language.call1((value,)) + } +} + +fn read_function<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&FunctionRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + let function: &FunctionRecord = owner + .functions() + .get(index) + .ok_or_else(|| PyIndexError::new_err(index))?; + read(function, py) +} + +fn function_classes<'py>( + owner: &Py, + py: Python<'py>, +) -> PyResult> { + owner + .bind(py) + .borrow() + .input_classes(py)? + .get_item(FUNCTION_CLASSES)? + .extract() +} + +/// The attributes a function's input and compiled views share. +macro_rules! function_getters { + ($view:ty, { $($extra:tt)* }) => { + #[pymethods] + impl $view { + $($extra)* + + #[getter] + fn name(&self, py: Python<'_>) -> PyResult { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.name.clone()) + }) + } + + #[getter] + fn arguments(&self, py: Python<'_>) -> PyResult> { + let classes: FunctionClasses<'_> = function_classes(&self.owner, py)?; + read_function(&self.owner, self.index, py, |function, _| { + FunctionRecord::named_types( + &function.attached.arguments, + &classes.function_argument, + &function.arguments, + ) + }) + } + + #[getter] + fn returns(&self, py: Python<'_>) -> PyResult { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.returns.clone()) + }) + } + + #[getter] + fn body_sql(&self, py: Python<'_>) -> PyResult { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.body.clone()) + }) + } + + #[getter] + fn return_columns(&self, py: Python<'_>) -> PyResult> { + let classes: FunctionClasses<'_> = function_classes(&self.owner, py)?; + read_function(&self.owner, self.index, py, |function, _| { + FunctionRecord::named_types( + &function.attached.return_columns, + &classes.function_return_column, + &function.return_columns, + ) + }) + } + + #[getter] + fn references(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_function(&self.owner, self.index, py, |function, _| { + reference_tuple( + &classes, + &function.attached.references, + &function.references, + ) + }) + } + + #[getter] + fn language<'py>(&self, py: Python<'py>) -> PyResult> { + let classes: FunctionClasses<'py> = function_classes(&self.owner, py)?; + read_function(&self.owner, self.index, py, |function, _| { + function.language(&classes) + }) + } + + #[getter] + fn runtime_version(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function + .attached + .python + .as_ref() + .and_then(|python| python.runtime_version.clone())) + }) + } + + #[getter] + fn entry_point(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function + .attached + .python + .as_ref() + .and_then(|python| python.entry_point.clone())) + }) + } + + #[getter] + fn packages<'py>(&self, py: Python<'py>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, py| { + let packages: &[String] = function + .attached + .python + .as_ref() + .map_or(&[], |python| python.packages.as_slice()); + PyTuple::new(py, packages) + }) + } + + #[getter] + fn tags<'py>(&self, py: Python<'py>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, py| { + PyTuple::new(py, &function.attached.tags) + }) + } + + #[getter] + fn description(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.description.clone()) + }) + } + } + }; +} + +/// One attached function, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeFunctionInput { + owner: Py, + index: usize, +} + +impl NativeFunctionInput { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +function_getters!(NativeFunctionInput, { + #[getter] + fn function_file(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, py| { + Ok(function.attached.function_file.clone_ref(py)) + }) + } + + #[getter] + fn database(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.destination.database.clone()) + }) + } + + #[getter] + fn schema(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.destination.schema.clone()) + }) + } + + #[getter] + fn logical_database(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.destination.logical_database.clone()) + }) + } + + #[getter] + fn logical_schema(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.destination.logical_schema.clone()) + }) + } + + #[getter] + fn fingerprint_database(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.fingerprint_destination.database.clone()) + }) + } + + #[getter] + fn fingerprint_schema(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.fingerprint_destination.schema.clone()) + }) + } + + #[getter] + fn fingerprint_logical_database(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function + .attached + .fingerprint_destination + .logical_database + .clone()) + }) + } + + #[getter] + fn fingerprint_logical_schema(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, _| { + Ok(function + .attached + .fingerprint_destination + .logical_schema + .clone()) + }) + } + + #[getter] + fn declaration_usages(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, py| { + Ok(function.attached.usages.clone_ref(py)) + }) + } + + /// `(node type, name, tags, references)`, what project assembly resolves dependencies from. + #[getter] + fn assembly_row(&self, py: Python<'_>) -> PyResult { + read_function(&self.owner, self.index, py, |function, _| { + let attached: &AttachedFunction = &function.attached; + Ok(( + attached.node_type(), + attached.name.clone(), + attached.tags.clone(), + attached + .references + .iter() + .map(|(kind, name, package, _)| (kind.clone(), name.clone(), package.clone())) + .collect(), + )) + }) + } +}); + +/// One assembled function, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeCompiledFunction { + owner: Py, + index: usize, + /// The destination and fingerprint destination with target defaults applied. + defaulted: Option<(DestinationFact, DestinationFact)>, + destination: PyOnceLock>, + fingerprint_destination: PyOnceLock>, +} + +impl NativeCompiledFunction { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self::defaulted(owner, index, None) + } + + fn defaulted( + owner: Py, + index: usize, + defaulted: Option<(DestinationFact, DestinationFact)>, + ) -> Self { + Self { + owner, + index, + defaulted, + destination: PyOnceLock::new(), + fingerprint_destination: PyOnceLock::new(), + } + } + + /// The destination and fingerprint destination, with target defaults once applied. + fn current_destinations(&self, py: Python<'_>) -> PyResult<(DestinationFact, DestinationFact)> { + if let Some(defaulted) = &self.defaulted { + return Ok(defaulted.clone()); + } + read_function(&self.owner, self.index, py, |function, _| { + Ok(( + function.attached.destination.clone(), + function.attached.fingerprint_destination.clone(), + )) + }) + } +} + +function_getters!(NativeCompiledFunction, { + /// The function's planner-facing `CompiledObjectKey`. + #[getter] + fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_function(&self.owner, self.index, py, |function, py| { + function + .key + .get_or_try_init(py, || { + object_key( + &classes.compiled, + function.attached.node_type(), + &function.attached.name, + ) + }) + .map(|key| key.clone_ref(py)) + }) + } + + /// The planner-facing keys of the resources the function reads. + #[getter] + fn deps<'py>(&self, py: Python<'py>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_function(&self.owner, self.index, py, |function, py| { + let deps: &[ObjectKeyRow] = function + .deps + .as_deref() + .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; + let mut keys: Vec> = Vec::with_capacity(deps.len()); + for (kind, name) in deps { + keys.push(object_key(&classes.compiled, kind, name)?); + } + PyTuple::new(py, keys) + }) + } + + #[getter] + fn relative_path(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, py| { + Ok(function + .attached + .function_file + .bind(py) + .getattr(FUNCTION_FILE_ATTRIBUTE)? + .unbind()) + }) + } + + #[getter] + fn source_file_path(&self, py: Python<'_>) -> PyResult> { + read_function(&self.owner, self.index, py, |function, py| { + Ok(function + .attached + .function_file + .bind(py) + .getattr(FILE_PATH_ATTRIBUTE)? + .unbind()) + }) + } + + /// The function's planner-facing `CompiledRelationLocation`. + #[getter] + fn destination(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.destination + .get_or_try_init(py, || { + relation_location(&classes.compiled, &self.current_destinations(py)?.0) + }) + .map(|destination| destination.clone_ref(py)) + } + + /// The destination the function's fingerprint reads. + #[getter] + fn fingerprint_destination(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.fingerprint_destination + .get_or_try_init(py, || { + relation_location(&classes.compiled, &self.current_destinations(py)?.1) + }) + .map(|destination| destination.clone_ref(py)) + } + + /// This function with target defaults; a non-inheriting Python function is name-qualified. + fn with_target_defaults( + &self, + py: Python<'_>, + defaults: (Option, Option), + render: Bound<'_, PyAny>, + python_functions_inherit_default_namespace: bool, + ) -> PyResult { + let python: bool = read_function(&self.owner, self.index, py, |function, _| { + Ok(function.attached.python.is_some()) + })?; + let apply: bool = !python || python_functions_inherit_default_namespace; + let (current, fingerprint) = self.current_destinations(py)?; + let fingerprint: DestinationFact = + defaulted_destination(&fingerprint, defaults.clone(), &render)?; + let defaults: (Option, Option) = + if apply { defaults } else { (None, None) }; + let mut defaulted: DestinationFact = defaulted_destination(¤t, defaults, &render)?; + if !apply && defaulted.qualified_name.is_none() { + defaulted.qualified_name = Some(defaulted.name.clone()); + } + Ok(Self::defaulted( + self.owner.clone_ref(py), + self.index, + Some((defaulted, fingerprint)), + )) + } +}); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs new file mode 100644 index 0000000000..0845ff43ef --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs @@ -0,0 +1,374 @@ +//! A compile's attached inputs, read from the native compiled project that owns its models. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyclass, pymethods}; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; + +const MODEL_INPUTS_FIELD: &str = "model_inputs"; +const RUN_ID_FIELD: &str = "run_id"; +const NO_SQL_VALIDATION_FIELD: &str = "no_sql_validation"; + +/// Compile inputs held as Python objects: user code, discovery or an integration owns them. +struct BoundaryInputs { + /// The discovered project config. + project_config: Py, + /// The discovered local config. + local_config: Py, + /// The discovered project inputs. + discovered_inputs: Py, + /// The adapter's lexical syntax. + sql_lexical_syntax: Py, + /// The selected target config, or `None`. + effective_target: Py, + /// The compile cache directory path. + compile_cache_dir: Py, + /// The resolved connection, adapter integration values. + effective_connection: Py, + /// Project variables, user values. + effective_vars: Py, + /// The macro context user macro bodies run with. + macro_context: Py, + /// Loaded macro modules, user code. + loaded_macros: Py, + /// Public enums, user declaration values. + public_enums: Py, + /// Public constants, user declaration values. + public_constants: Py, + /// The external reference resolver integration, or `None`. + external_sql_reference_resolver: Py, +} + +/// Compiler-derived inputs still held as Python objects until they are owner facts. +struct CompilerHandles { + /// Effective settings. + effective_settings: Py, + /// Attached seed inputs. + seed_inputs: Py, + /// Attached source inputs. + source_inputs: Py, + /// Attached SQL function inputs. + sql_function_inputs: Py, + /// Attached SQL test inputs. + test_inputs: Py, + /// Attached SQL scenario inputs. + scenario_inputs: Py, + /// Attached audit inputs. + audit_inputs: Py, + /// Compiler diagnostics. + diagnostics: Py, + /// The declaration scope index. + scope_index: Py, + /// The declaration scope build. + declaration_scope: Py, +} + +/// The values of one compile inputs view. +struct InputsFields { + run_id: String, + effective_target_name: Option, + no_sql_validation: bool, + model_inputs: Py, + boundary: BoundaryInputs, + compiler: CompilerHandles, +} + +impl InputsFields { + /// The fields named in `fields`, each required. + fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + run_id: required(fields, RUN_ID_FIELD)?.extract()?, + effective_target_name: required(fields, "effective_target_name")?.extract()?, + no_sql_validation: required(fields, NO_SQL_VALIDATION_FIELD)?.extract()?, + model_inputs: required(fields, MODEL_INPUTS_FIELD)? + .downcast_into::()? + .unbind(), + boundary: BoundaryInputs { + project_config: required(fields, "project_config")?.unbind(), + local_config: required(fields, "local_config")?.unbind(), + discovered_inputs: required(fields, "discovered_inputs")?.unbind(), + sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), + effective_target: required(fields, "effective_target")?.unbind(), + compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), + effective_connection: required(fields, "effective_connection")?.unbind(), + effective_vars: required(fields, "effective_vars")?.unbind(), + macro_context: required(fields, "macro_context")?.unbind(), + loaded_macros: required(fields, "loaded_macros")?.unbind(), + public_enums: required(fields, "public_enums")?.unbind(), + public_constants: required(fields, "public_constants")?.unbind(), + external_sql_reference_resolver: required( + fields, + "external_sql_reference_resolver", + )? + .unbind(), + }, + compiler: CompilerHandles { + effective_settings: required(fields, "effective_settings")?.unbind(), + seed_inputs: required(fields, "seed_inputs")?.unbind(), + source_inputs: required(fields, "source_inputs")?.unbind(), + sql_function_inputs: required(fields, "sql_function_inputs")?.unbind(), + test_inputs: required(fields, "test_inputs")?.unbind(), + scenario_inputs: required(fields, "scenario_inputs")?.unbind(), + audit_inputs: required(fields, "audit_inputs")?.unbind(), + diagnostics: required(fields, "diagnostics")?.unbind(), + scope_index: required(fields, "scope_index")?.unbind(), + declaration_scope: required(fields, "declaration_scope")?.unbind(), + }, + }) + } + + /// Every field by name. + fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(RUN_ID_FIELD, &self.run_id)?; + fields.set_item("effective_target_name", &self.effective_target_name)?; + fields.set_item(NO_SQL_VALIDATION_FIELD, self.no_sql_validation)?; + fields.set_item(MODEL_INPUTS_FIELD, self.model_inputs.bind(py))?; + fields.set_item("project_config", self.boundary.project_config.bind(py))?; + fields.set_item("local_config", self.boundary.local_config.bind(py))?; + fields.set_item( + "discovered_inputs", + self.boundary.discovered_inputs.bind(py), + )?; + fields.set_item( + "sql_lexical_syntax", + self.boundary.sql_lexical_syntax.bind(py), + )?; + fields.set_item("effective_target", self.boundary.effective_target.bind(py))?; + fields.set_item( + "compile_cache_dir", + self.boundary.compile_cache_dir.bind(py), + )?; + fields.set_item( + "effective_connection", + self.boundary.effective_connection.bind(py), + )?; + fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; + fields.set_item("macro_context", self.boundary.macro_context.bind(py))?; + fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; + fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; + fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; + fields.set_item( + "external_sql_reference_resolver", + self.boundary.external_sql_reference_resolver.bind(py), + )?; + fields.set_item( + "effective_settings", + self.compiler.effective_settings.bind(py), + )?; + fields.set_item("seed_inputs", self.compiler.seed_inputs.bind(py))?; + fields.set_item("source_inputs", self.compiler.source_inputs.bind(py))?; + fields.set_item( + "sql_function_inputs", + self.compiler.sql_function_inputs.bind(py), + )?; + fields.set_item("test_inputs", self.compiler.test_inputs.bind(py))?; + fields.set_item("scenario_inputs", self.compiler.scenario_inputs.bind(py))?; + fields.set_item("audit_inputs", self.compiler.audit_inputs.bind(py))?; + fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; + fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; + fields.set_item( + "declaration_scope", + self.compiler.declaration_scope.bind(py), + )?; + Ok(fields) + } +} + +fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { + fields + .get_item(name)? + .ok_or_else(|| PyTypeError::new_err(format!("missing compile input field {name}"))) +} + +/// One compile's attached inputs, its model inputs projected from the owner. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeInputsView { + owner: Py, + fields: InputsFields, +} + +impl NativeInputsView { + /// The view over `owner` with these input fields. + pub(crate) fn new( + owner: Py, + fields: &Bound<'_, PyDict>, + ) -> PyResult { + Ok(Self { + owner, + fields: InputsFields::from_fields(fields)?, + }) + } +} + +#[pymethods] +impl NativeInputsView { + /// These inputs with `changes` applied; the owner's model inputs are shared. + #[pyo3(signature = (**changes))] + fn with_changes(&self, py: Python<'_>, changes: Option>) -> PyResult { + let current: Bound<'_, PyDict> = self.fields.as_fields(py)?; + if let Some(changes) = changes { + for (name, value) in changes.iter() { + if !current.contains(&name)? { + return Err(PyTypeError::new_err(format!( + "unknown compile input field {name}" + ))); + } + current.set_item(name, value)?; + } + } + Self::new(self.owner.clone_ref(py), ¤t) + } + + /// Every field by name, for an explicit `CompileProjectInputs` projection. + fn fields<'py>(&self, py: Python<'py>) -> PyResult> { + let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; + fields.set_item("native_project", self.owner.bind(py))?; + Ok(fields) + } + + #[getter] + fn run_id(&self) -> String { + self.fields.run_id.clone() + } + + #[getter] + fn effective_target_name(&self) -> Option { + self.fields.effective_target_name.clone() + } + + #[getter(no_sql_validation)] + fn is_sql_validation_skipped(&self) -> bool { + self.fields.no_sql_validation + } + + #[getter] + fn model_inputs(&self, py: Python<'_>) -> Py { + self.fields.model_inputs.clone_ref(py) + } + + #[getter] + fn native_project(&self, py: Python<'_>) -> Py { + self.owner.clone_ref(py) + } + + #[getter] + fn project_config(&self, py: Python<'_>) -> Py { + self.fields.boundary.project_config.clone_ref(py) + } + + #[getter] + fn local_config(&self, py: Python<'_>) -> Py { + self.fields.boundary.local_config.clone_ref(py) + } + + #[getter] + fn discovered_inputs(&self, py: Python<'_>) -> Py { + self.fields.boundary.discovered_inputs.clone_ref(py) + } + + #[getter] + fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { + self.fields.boundary.sql_lexical_syntax.clone_ref(py) + } + + #[getter] + fn effective_target(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_target.clone_ref(py) + } + + #[getter] + fn compile_cache_dir(&self, py: Python<'_>) -> Py { + self.fields.boundary.compile_cache_dir.clone_ref(py) + } + + #[getter] + fn effective_connection(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_connection.clone_ref(py) + } + + #[getter] + fn effective_vars(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_vars.clone_ref(py) + } + + #[getter] + fn macro_context(&self, py: Python<'_>) -> Py { + self.fields.boundary.macro_context.clone_ref(py) + } + + #[getter] + fn loaded_macros(&self, py: Python<'_>) -> Py { + self.fields.boundary.loaded_macros.clone_ref(py) + } + + #[getter] + fn public_enums(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_enums.clone_ref(py) + } + + #[getter] + fn public_constants(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_constants.clone_ref(py) + } + + #[getter] + fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { + self.fields + .boundary + .external_sql_reference_resolver + .clone_ref(py) + } + + #[getter] + fn effective_settings(&self, py: Python<'_>) -> Py { + self.fields.compiler.effective_settings.clone_ref(py) + } + + #[getter] + fn seed_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.seed_inputs.clone_ref(py) + } + + #[getter] + fn source_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.source_inputs.clone_ref(py) + } + + #[getter] + fn sql_function_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.sql_function_inputs.clone_ref(py) + } + + #[getter] + fn test_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.test_inputs.clone_ref(py) + } + + #[getter] + fn scenario_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.scenario_inputs.clone_ref(py) + } + + #[getter] + fn audit_inputs(&self, py: Python<'_>) -> Py { + self.fields.compiler.audit_inputs.clone_ref(py) + } + + #[getter] + fn diagnostics(&self, py: Python<'_>) -> Py { + self.fields.compiler.diagnostics.clone_ref(py) + } + + #[getter] + fn scope_index(&self, py: Python<'_>) -> Py { + self.fields.compiler.scope_index.clone_ref(py) + } + + #[getter] + fn declaration_scope(&self, py: Python<'_>) -> Py { + self.fields.compiler.declaration_scope.clone_ref(py) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs new file mode 100644 index 0000000000..aaf6b3a972 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -0,0 +1,33 @@ +pub(crate) mod audit_loop; +pub(crate) mod audit_views; +pub(crate) mod compiled_model_analysis; +pub(crate) mod compiled_model_view; +pub(crate) mod function_loop; +pub(crate) mod function_views; +pub(crate) mod inputs_view; +pub(crate) mod model_input_view; +pub(crate) mod model_loop; +pub(crate) mod model_loop_calls; +pub(crate) mod model_loop_config; +pub(crate) mod model_loop_contracts; +pub(crate) mod model_loop_external; +pub(crate) mod model_loop_header; +pub(crate) mod model_loop_hooks; +pub(crate) mod model_loop_host; +pub(crate) mod model_loop_inputs; +pub(crate) mod model_loop_macros; +pub(crate) mod model_loop_named; +pub(crate) mod model_loop_opt_out; +pub(crate) mod model_loop_python_hooks; +pub(crate) mod model_loop_schema; +pub(crate) mod model_loop_visibility; +pub(crate) mod project; +pub(crate) mod project_view; +pub(crate) mod registration; +pub(crate) mod resource_diagnostics; +pub(crate) mod scenario_loop; +pub(crate) mod seed_views; +pub(crate) mod source_loop; +pub(crate) mod source_views; +pub(crate) mod sql_test_loop; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs new file mode 100644 index 0000000000..dc0789f477 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -0,0 +1,449 @@ +//! A model's compile input, read from the native compiled project that owns it. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{FromPyObject, IntoPyObject, pyclass, pymethods}; + +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + CompiledArguments, CompiledRecord, CompiledViewClasses, NativeCompiledModel, compiled_record, +}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::render::declaration_expansion::SpanRow; +use crate::bindings::_helpers::rules::relation_keys::{RelationKeyRow, relation_key_row}; + +const MODEL_RELATION_KIND: &str = "ref"; + +/// The Python classes a model input's projected attributes are built with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct InputViewClasses<'py> { + pub(crate) compiled: CompiledViewClasses<'py>, + sql_reference: Bound<'py, PyAny>, + reference_kind: Bound<'py, PyAny>, + schema_model_entry: Bound<'py, PyAny>, + sql_expansion: Bound<'py, PyAny>, + expansion_span: Bound<'py, PyAny>, +} + +/// A model's attached header schema; its specs carry user-authored audit arguments. +pub(crate) struct SchemaRecord { + pub(crate) description: Option, + pub(crate) columns: Py, + pub(crate) audits: Py, + pub(crate) dynamic_columns: Py, + pub(crate) type_enforcement: bool, + pub(crate) model_schema: Option, +} + +/// The expansion passes of a model's query, from authored text to expanded SQL. +pub(crate) struct ExpansionRecord { + /// The authored query below the header. + pub(crate) authored_sql: Py, + /// The declaration- and macro-expanded SQL before canonicalisation. + pub(crate) expanded_sql: Py, + pub(crate) interpolation_spans: Vec, + pub(crate) declaration_spans: Vec, + pub(crate) macro_spans: Vec, + /// Whether interpolation left the authored query unchanged. + pub(crate) authored: bool, +} + +/// What the model loop finished for one model beyond its facts; handles carry user values. +pub(crate) struct ModelInputRecord { + /// The discovered model file, owned by the discovery inputs. + pub(crate) model_file: Py, + /// The resolved config, whose values hold user values. + pub(crate) config: Py, + /// Immutable text shared with discovery, expansions and every Python-facing projection. + pub(crate) query_sql: Py, + pub(crate) macro_source_sql: Py, + pub(crate) references: Vec, + pub(crate) schema: Option, + pub(crate) sql_validation_enabled: bool, + pub(crate) rejected_opt_out: Option>, + /// The model's own enums and constants, whose values are user values. + pub(crate) enum_declarations: Py, + pub(crate) constant_declarations: Py, + pub(crate) enum_columns: Py, + pub(crate) macro_deps: Vec, + /// Usage records, the identities the scope index is built from. + pub(crate) macro_usages: Py, + pub(crate) declaration_usages: Py, + pub(crate) expansion: ExpansionRecord, + /// Attribute projections built on first access, shared by every view of the model. + pub(crate) projections: InputProjections, + /// What assembly and the semantic checks finished, once the model is assembled. + pub(crate) compiled: Option, +} + +/// A model input's Python attribute projections, each built once on first access. +pub(crate) struct InputProjections { + references: PyOnceLock>, + schema_entry: PyOnceLock>, + sql_expansion: PyOnceLock>, + lint_expansion: PyOnceLock>, +} + +impl Default for InputProjections { + fn default() -> Self { + Self { + references: PyOnceLock::new(), + schema_entry: PyOnceLock::new(), + sql_expansion: PyOnceLock::new(), + lint_expansion: PyOnceLock::new(), + } + } +} + +/// One model's compile input, projected from the owner on demand. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeModelInput { + owner: Py, + index: usize, +} + +impl NativeModelInput { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } + + fn read<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, + ) -> PyResult { + read_input(&self.owner, self.index, py, read) + } +} + +/// Read the input record at `index` of `owner`. +pub(crate) fn read_input<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + read(owner.input(index)?, py) +} + +/// The classes the owner's views project attributes with. +pub(crate) fn view_classes<'py>( + owner: &Py, + py: Python<'py>, +) -> PyResult> { + owner.bind(py).borrow().input_classes(py)?.extract() +} + +#[pymethods] +impl NativeModelInput { + #[getter] + fn model_file(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.model_file.clone_ref(py))) + } + + #[getter] + fn config(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.config.clone_ref(py))) + } + + #[getter] + fn query_sql(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.query_sql.clone_ref(py))) + } + + #[getter] + fn macro_source_sql(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.macro_source_sql.clone_ref(py))) + } + + #[getter] + fn references(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| references(&classes, record)) + } + + #[getter] + fn schema_entry(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| schema_entry(&classes, record)) + } + + /// Each reference as `(kind, name, package, call argument count)`. + #[getter] + fn reference_rows(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, _| Ok(record.references.clone())) + } + + /// The model's declared key facts, for deriving its unique keys. + #[getter] + fn relation_key_row(&self, py: Python<'_>) -> PyResult { + self.read(py, model_relation_key_row) + } + + /// The name of the schema entry attached to the model, or `None`. + #[getter] + fn attached_schema_name(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| attached_schema_name(record, py)) + } + + #[getter] + fn schema_file(&self, py: Python<'_>) -> Py { + py.None() + } + + #[getter] + fn sql_validation_enabled(&self, py: Python<'_>) -> PyResult { + self.read(py, |record, _| Ok(record.sql_validation_enabled)) + } + + #[getter] + fn rejected_sql_analysis_opt_out(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(rejected_opt_out(record, py))) + } + + #[getter] + fn enum_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.enum_declarations.clone_ref(py))) + } + + #[getter] + fn constant_declarations(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| { + Ok(record.constant_declarations.clone_ref(py)) + }) + } + + #[getter] + fn enum_columns(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.enum_columns.clone_ref(py))) + } + + #[getter] + fn macro_deps<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| PyTuple::new(py, &record.macro_deps)) + } + + #[getter] + fn macro_usages(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.macro_usages.clone_ref(py))) + } + + #[getter] + fn declaration_usages(&self, py: Python<'_>) -> PyResult> { + self.read(py, |record, py| Ok(record.declaration_usages.clone_ref(py))) + } + + /// Record what assembly finished for the model; return its compiled model view. + fn compiled_model( + &self, + py: Python<'_>, + arguments: CompiledArguments, + ) -> PyResult { + let mut owner = self.owner.bind(py).borrow_mut(); + let record: &mut ModelInputRecord = owner.input_mut(self.index)?; + let name: String = record + .model_file + .bind(py) + .getattr("file_path")? + .getattr("stem")? + .extract()?; + record.compiled = Some(compiled_record(py, record, &name, arguments)?); + Ok(NativeCompiledModel::new( + self.owner.clone_ref(py), + self.index, + )) + } + + /// The authored-to-expanded passes, when interpolation left the authored query unchanged. + #[getter] + fn sql_expansion(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, _| authored_expansion(&classes, record)) + } + + /// The whole expansion, variables included, which early SQL lint reuses. + #[getter] + fn lint_expansion(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.read(py, |record, py| { + record + .projections + .lint_expansion + .get_or_try_init(py, || { + let expansion: &ExpansionRecord = &record.expansion; + expansion_object( + &classes, + expansion, + &[ + &expansion.interpolation_spans, + &expansion.declaration_spans, + &expansion.macro_spans, + ], + ) + }) + .map(|expansion| expansion.clone_ref(py)) + }) + } +} + +/// The authored-to-expanded passes, when interpolation left the authored query unchanged. +pub(crate) fn authored_expansion( + classes: &InputViewClasses<'_>, + record: &ModelInputRecord, +) -> PyResult> { + let py: Python<'_> = classes.sql_expansion.py(); + record + .projections + .sql_expansion + .get_or_try_init(py, || { + let expansion: &ExpansionRecord = &record.expansion; + if !expansion.authored { + return Ok(py.None()); + } + expansion_object( + classes, + expansion, + &[&expansion.declaration_spans, &expansion.macro_spans], + ) + }) + .map(|expansion| expansion.clone_ref(py)) +} + +/// Where a rejected `sql_analysis false` was authored, or `None`. +pub(crate) fn rejected_opt_out(record: &ModelInputRecord, py: Python<'_>) -> Py { + record + .rejected_opt_out + .as_ref() + .map_or_else(|| py.None(), |location| location.clone_ref(py)) +} + +/// The model's `CompileSqlReference` tuple, built once. +pub(crate) fn references( + classes: &InputViewClasses<'_>, + record: &ModelInputRecord, +) -> PyResult> { + reference_tuple(classes, &record.references, &record.projections.references) +} + +/// `CompileSqlReference` records of `rows`, built once into `cache`. +pub(crate) fn reference_tuple( + classes: &InputViewClasses<'_>, + rows: &[ReferenceRow], + cache: &PyOnceLock>, +) -> PyResult> { + let py: Python<'_> = classes.sql_reference.py(); + cache + .get_or_try_init(py, || { + let mut references: Vec> = Vec::with_capacity(rows.len()); + for (kind, name, package, count) in rows { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("ref_kind", classes.reference_kind.call1((kind,))?)?; + fields.set_item("ref_name", name)?; + fields.set_item("ref_package", package)?; + fields.set_item("call_argument_count", count)?; + references.push(classes.sql_reference.call((), Some(&fields))?); + } + Ok::<_, pyo3::PyErr>(PyTuple::new(py, references)?.unbind()) + }) + .map(|references| references.clone_ref(py)) +} + +/// The model's declared key facts: its config and attached schema columns and audits. +pub(crate) fn model_relation_key_row( + record: &ModelInputRecord, + py: Python<'_>, +) -> PyResult { + let values: Bound<'_, PyAny> = record.config.bind(py).getattr("values")?; + relation_key_row( + MODEL_RELATION_KIND, + record + .model_file + .bind(py) + .getattr("file_path")? + .getattr("stem")? + .extract()?, + values.downcast::()?, + record.schema.as_ref().map(|schema| { + ( + schema.columns.bind(py).as_any(), + schema.audits.bind(py).as_any(), + ) + }), + ) +} + +/// The name of the model's attached schema entry, its file stem, or `None`. +pub(crate) fn attached_schema_name( + record: &ModelInputRecord, + py: Python<'_>, +) -> PyResult> { + if record.schema.is_none() { + return Ok(None); + } + Ok(Some( + record + .model_file + .bind(py) + .getattr("file_path")? + .getattr("stem")? + .extract()?, + )) +} + +/// The model's attached `SchemaModelEntry`, or `None`, built once. +pub(crate) fn schema_entry( + classes: &InputViewClasses<'_>, + record: &ModelInputRecord, +) -> PyResult> { + let py: Python<'_> = classes.schema_model_entry.py(); + record + .projections + .schema_entry + .get_or_try_init(py, || { + let Some(schema) = &record.schema else { + return Ok::<_, pyo3::PyErr>(py.None()); + }; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item( + "name", + record + .model_file + .bind(py) + .getattr("file_path")? + .getattr("stem")?, + )?; + fields.set_item("model_schema", &schema.model_schema)?; + fields.set_item("description", &schema.description)?; + fields.set_item("type_enforcement", schema.type_enforcement.then_some(true))?; + fields.set_item("columns", schema.columns.bind(py))?; + fields.set_item("dynamic_columns", schema.dynamic_columns.bind(py))?; + fields.set_item("audits", schema.audits.bind(py))?; + Ok(classes.schema_model_entry.call((), Some(&fields))?.unbind()) + }) + .map(|entry| entry.clone_ref(py)) +} + +/// `CompiledSqlExpansion(authored_sql, expanded_sql, passes)` for these passes. +fn expansion_object( + classes: &InputViewClasses<'_>, + expansion: &ExpansionRecord, + passes: &[&Vec], +) -> PyResult> { + let py: Python<'_> = classes.sql_expansion.py(); + let mut pass_objects: Vec> = Vec::with_capacity(passes.len()); + for spans in passes { + let mut span_objects: Vec> = Vec::with_capacity(spans.len()); + for span in spans.iter() { + span_objects.push(classes.expansion_span.call1(span.into_pyobject(py)?)?); + } + pass_objects.push(PyTuple::new(py, span_objects)?); + } + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("authored_sql", expansion.authored_sql.bind(py))?; + fields.set_item("expanded_sql", expansion.expanded_sql.bind(py))?; + fields.set_item("passes", PyTuple::new(py, pass_objects)?)?; + Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs new file mode 100644 index 0000000000..14b80a697b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -0,0 +1,879 @@ +//! The compile-input model loop: each model's steps run in the attachment's order, natively. + +use crate::bindings::_helpers::compiled_project::audit_loop::{ + AttachmentOwner, AuditFileRow, AuditSettings, attached_audits, +}; +use crate::bindings::_helpers::compiled_project::audit_views::AttachedAudit; +use crate::bindings::_helpers::compiled_project::function_loop::{ + FunctionRow, FunctionSettings, attached_functions, checked_table_function_arities, +}; +use crate::bindings::_helpers::compiled_project::function_views::AttachedFunction; +use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, ModelConfigs}; +use crate::bindings::_helpers::compiled_project::model_loop_external::external_errors; +use crate::bindings::_helpers::compiled_project::model_loop_header::{ + AttachedHeader, HeaderSource, header_schema, +}; +use crate::bindings::_helpers::compiled_project::model_loop_hooks::{ + ExpandedHooks, HookModel, SqlHookClasses, expanded_hooks, +}; +use crate::bindings::_helpers::compiled_project::model_loop_host::{ + FinishedModel, InterpolationRow, ModelLoopRow, RecordClasses, RecordInputs, ReferenceRow, + finished_record, +}; +use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ + analysis_override, hook_presence, reference_row, +}; +use crate::bindings::_helpers::compiled_project::model_loop_macros::{ + ExpandedMacros, MacroClasses, MacroModel, expanded_macros, +}; +use crate::bindings::_helpers::compiled_project::model_loop_named::{ + SCHEMA_DECLARATION, named_usages, +}; +use crate::bindings::_helpers::compiled_project::model_loop_opt_out::{ + OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, +}; +use crate::bindings::_helpers::compiled_project::model_loop_python_hooks::{ + HookClasses, HookSignatures, +}; +use crate::bindings::_helpers::compiled_project::model_loop_schema::{ + ModelSchema, model_schema, schema_column_names, +}; +use crate::bindings::_helpers::compiled_project::model_loop_visibility::{ + ModelVisibility, VisibilityClasses, VisibilityModel, +}; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::scenario_loop::ScenarioRow; +use crate::bindings::_helpers::compiled_project::source_loop::{ + SourceRow, SourceSettings, attached_sources, +}; +use crate::bindings::_helpers::compiled_project::source_views::{ + AttachedSource, NativeSourceInput, retain_sources, +}; +use crate::bindings::_helpers::compiled_project::sql_test_loop::{ + AttachedSqlTests, SqlTestSettings, TestFileRow, TestScope, +}; +use crate::bindings::_helpers::discovery::project_files::python_semantics; +use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; +use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; +use crate::bindings::_helpers::render::declaration_expansion::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, + expand_declarations, +}; +use crate::bindings::_helpers::render::macro_call_rows::span_row; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::{FromPyObject, pyclass, pymethods}; +use sqlbuild_analysis::compiled_project::main::analysis_gate::analysis_gate; +use sqlbuild_analysis::compiled_project::main::calls_macros::calls_macros; +use sqlbuild_analysis::compiled_project::main::legacy_schema_error::legacy_schema_error; +use sqlbuild_analysis::compiled_project::main::located_scan_error::located_scan_error; +use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge_call_site_references; +use sqlbuild_analysis::compiled_project::models::{ + AnalysisGate, LegacySchemaFile, ModelLoopSettings, ScanFaultOrigin, SplicedSpan, +}; +use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::validated_model_intrinsics; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; +use sqlbuild_attachments::functions::models::KnownResources; +use sqlbuild_attachments::test_targets::models::TargetCatalog; +use sqlbuild_core::text::models::PythonText; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use std::collections::{HashMap, HashSet}; + +const MATERIALIZED_KEY: &str = "materialized"; +const CURSOR_KEY: &str = "cursor"; +const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +const VALUES_ATTRIBUTE: &str = "values"; +const RETENTION_ATTRIBUTE: &str = "time_travel_retention"; +const UNMANAGED_ATTRIBUTE: &str = "unmanaged"; +const TABLE_TYPE_ATTRIBUTE: &str = "table_type"; +const DECLARED_ATTRIBUTE: &str = "declared"; +const VALIDATE_METHOD: &str = "validate"; +const FILE_PATH_ARGUMENT: &str = "file_path"; +const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; +const HELP_ARGUMENT: &str = "help"; +const ERROR_ARGUMENT: &str = "error"; +const VALUES_ARGUMENT: &str = "values"; +const CODE_ARGUMENT: &str = "code"; +const ANALYSIS_CONFLICT: &str = "MODEL sql_analysis conflicts with legacy sql_validation"; +const ANALYSIS_CONFLICT_CODE: &str = "P003"; + +/// `(finished, config builds, projected visibilities, hook statements, described models)`. +type LoopCounts = (usize, usize, usize, usize, usize); +/// The native validator, header contract classes, and the loop's Python classes. +type LoopChecks = (Py, Py, Py); +/// `(analysis switches, native syntax, (project config path, legacy schema files))`. +type LoopSettingsInput = ( + (bool, bool, bool, bool), + (String, Vec<(String, Vec)>), +); + +/// The Python classes and error factory the loop builds usages, hooks and errors with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct LoopClasses<'py> { + pub(crate) usage_record: Bound<'py, PyAny>, + pub(crate) compile_error: Bound<'py, PyAny>, + pub(crate) sql_hook_entry: Bound<'py, PyAny>, + /// `native_config_error`: the exception a native config error raises as. + pub(crate) config_error: Bound<'py, PyAny>, + /// Discovered audit factories by name. + pub(crate) audit_factories: Bound<'py, PyDict>, + /// `UsageKind.GENERATED`, for usages an enum-typed column generates. + pub(crate) generated_usage: Bound<'py, PyAny>, + /// `ContractPolicy.ENFORCED`. + pub(crate) contract_enforced: Bound<'py, PyAny>, + /// Public reusable model schemas by name. + pub(crate) model_schemas: Bound<'py, PyDict>, + /// Each indexed schema's and Python hook's declaration record, by `(kind, name)`. + pub(crate) named_records: Bound<'py, PyDict>, + /// Project enums by name, as a schema's column types resolve them. + pub(crate) schema_enums: Bound<'py, PyDict>, + /// `CompilerDiagnostic` and `report_compile_diagnostic`, for S006. + pub(crate) compiler_diagnostic: Bound<'py, PyAny>, + pub(crate) report_diagnostic: Bound<'py, PyAny>, + /// `pathlib.Path`. + pub(crate) path_class: Bound<'py, PyAny>, + /// The dbt manifest's `ExternalSqlReferenceResolver`, or `None`. + pub(crate) external_resolver: Bound<'py, PyAny>, + /// `SqlReferenceKind`. + pub(crate) reference_kind: Bound<'py, PyAny>, + /// `functools.partial`, binding the model's file to its constant renderer. + pub(crate) partial: Bound<'py, PyAny>, + /// What a model's effective config is built from. + pub(crate) config: ConfigClasses<'py>, + /// What a model's visible declarations are resolved from. + pub(crate) visibility: VisibilityClasses<'py>, + /// What a model's Python hooks are checked against. + pub(crate) hooks: HookClasses<'py>, + /// What each model's SQL hooks are expanded with. + pub(crate) sql_hooks: SqlHookClasses<'py>, + /// What each model's macro calls are expanded with. + pub(crate) macros: MacroClasses<'py>, + /// `sql_analysis_opt_out_help(model_owned=True)`, the help of a model's syntax error. + pub(crate) syntax_help: String, + /// The same help for SQL a project's other resources own. + pub(crate) source_syntax_help: String, + /// `InvalidScopePathError`. + pub(crate) scope_path_error: Bound<'py, PyAny>, + /// What each finished model's input record is built with. + pub(crate) record: RecordClasses<'py>, + /// What model input views project their attributes with. + pub(crate) input_view: Bound<'py, PyAny>, +} + +/// The ordered model loop of one compile-input stage. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeModelLoop { + pub(crate) validator: Py, + pub(crate) syntax: LexicalSyntax, + pub(crate) reserved_markers: Vec, + pub(crate) python: PythonText, + pub(crate) settings: ModelLoopSettings, + pub(crate) project_config_path: String, + legacy_schema_files: Vec, + contract_classes: Py, + loop_classes: Py, +} + +/// What one run reads beside each model: the host, Python classes and header contract classes. +pub(crate) struct LoopContext<'a, 'py> { + pub(crate) host: &'a Bound<'py, PyAny>, + pub(crate) classes: &'a LoopClasses<'py>, + pub(crate) contracts: &'a ContractClasses<'py>, +} + +/// The declarations a model sees and the identity its usages are recorded for. +pub(crate) struct VisibleDeclarations<'py> { + pub(crate) declarations: Bound<'py, PyAny>, + pub(crate) consumer: Bound<'py, PyAny>, +} + +/// What the model loop knows about a model before finishing it. +struct LoopModel<'a, 'py> { + row: &'a ModelLoopRow, + config: Bound<'py, PyAny>, + visible: &'a VisibleDeclarations<'py>, + signatures: &'a HookSignatures, + visibility: &'a ModelVisibility<'py>, +} + +#[pymethods] +impl NativeModelLoop { + #[new] + fn new( + checks: LoopChecks, + syntax: LexicalSyntaxInput, + cursor: (Vec, ((u8, u8), String)), + settings: LoopSettingsInput, + ) -> PyResult { + let (validator, contract_classes, loop_classes) = checks; + let (reserved_markers, (python_version, unicode_version)) = cursor; + let ( + (sql_analysis, no_sql_validation, defer_sql_validation, require_sql_analysis), + (project_config_path, legacy_schema_files), + ) = settings; + Ok(Self { + validator, + syntax: syntax.into(), + reserved_markers, + python: python_semantics(python_version, &unicode_version)?, + settings: ModelLoopSettings { + sql_analysis, + no_sql_validation, + defer_sql_validation, + require_sql_analysis, + }, + project_config_path, + legacy_schema_files: legacy_schema_files + .into_iter() + .map(|(relative_path, model_names)| LegacySchemaFile { + relative_path, + model_names, + }) + .collect(), + contract_classes, + loop_classes, + }) + } + + /// Build every model in order into `project`; return each model's input view and counts. + fn run<'py>( + slf: &Bound<'py, Self>, + project: Bound<'py, NativeCompiledProject>, + host: Bound<'py, PyAny>, + models: Vec, + ) -> PyResult<(Vec>, LoopCounts)> { + let py: Python<'py> = slf.py(); + project + .borrow_mut() + .keep_attachment_loop(slf.clone().unbind(), host.clone().unbind()); + slf.get().run_models(py, project, host, models) + } +} + +impl NativeModelLoop { + fn run_models<'py>( + &self, + py: Python<'py>, + project: Bound<'py, NativeCompiledProject>, + host: Bound<'py, PyAny>, + models: Vec, + ) -> PyResult<(Vec>, LoopCounts)> { + let contracts: ContractClasses<'py> = self.contract_classes.bind(py).extract()?; + let classes: LoopClasses<'py> = self.loop_classes.bind(py).extract()?; + let context: LoopContext<'_, 'py> = LoopContext { + host: &host, + classes: &classes, + contracts: &contracts, + }; + project + .borrow_mut() + .set_input_classes(classes.input_view.clone().unbind()); + let mut inputs: Vec> = Vec::with_capacity(models.len()); + let mut finished_natively: usize = 0; + let mut hook_statements: usize = 0; + let mut described: usize = 0; + let configs: ModelConfigs<'py> = ModelConfigs::default(); + let visibility: ModelVisibility<'py> = ModelVisibility::default(); + let signatures: HookSignatures = HookSignatures::default(); + for row in models { + let ModelLoopRow((name, _, relative_path, file_path), header, _, _, _, _, _) = &row; + let (_, header_values, enums, constants) = header; + let visible: VisibleDeclarations<'py> = visibility.for_model( + &context, + &VisibilityModel { + name, + file_path, + local: (enums.bind(py), constants.bind(py)), + }, + )?; + let config: Bound<'py, PyAny> = + configs.effective(&context, header_values.bind(py), (name, relative_path))?; + let model: LoopModel<'_, 'py> = LoopModel { + row: &row, + config, + visible: &visible, + signatures: &signatures, + visibility: &visibility, + }; + let finished: FinishedModel<'py> = self.finish(py, &context, model)?; + finished_natively += 1; + hook_statements += finished.hook_statements; + described += usize::from(finished.header.schema.is_some()); + let record_inputs: RecordInputs<'_, 'py> = RecordInputs { + classes: &classes.record, + visible: &visible.declarations, + }; + let (record, facts) = finished_record(&record_inputs, row, finished)?; + let mut owner = project.borrow_mut(); + owner.retain(facts); + let index: usize = owner.retain_input(record); + drop(owner); + inputs.push(Bound::new( + py, + NativeModelInput::new(project.clone().unbind(), index), + )?); + } + Ok(( + inputs, + ( + finished_natively, + configs.builds(), + visibility.projected(), + hook_statements, + described, + ), + )) + } +} + +impl NativeModelLoop { + /// Attach every discovered source in order into `project`; return each source's view. + pub(crate) fn run_sources<'py>( + &self, + project: &Bound<'py, NativeCompiledProject>, + host: &Bound<'py, PyAny>, + rows: Vec, + settings: SourceSettings, + ) -> PyResult> { + let py: Python<'py> = host.py(); + let contracts: ContractClasses<'py> = self.contract_classes.bind(py).extract()?; + let classes: LoopClasses<'py> = self.loop_classes.bind(py).extract()?; + let context: LoopContext<'_, 'py> = LoopContext { + host, + classes: &classes, + contracts: &contracts, + }; + let attached: Vec = attached_sources(self, &context, rows, settings)?; + Ok(retain_sources(project, attached)) + } + + /// Attach every singular audit, then each owner's attached audits. + pub(crate) fn run_audits<'py>( + &self, + host: &Bound<'py, PyAny>, + files: &[AuditFileRow], + owners: Vec>, + settings: &AuditSettings, + ) -> PyResult<(Vec, Vec>)> { + let py: Python<'py> = host.py(); + let contracts: ContractClasses<'py> = self.contract_classes.bind(py).extract()?; + let classes: LoopClasses<'py> = self.loop_classes.bind(py).extract()?; + let context: LoopContext<'_, 'py> = LoopContext { + host, + classes: &classes, + contracts: &contracts, + }; + attached_audits(self, &context, (files, owners), settings) + } + + /// Attach every SQL test case, then every scenario, against the project's resources. + pub(crate) fn run_sql_tests<'py>( + &self, + host: &Bound<'py, PyAny>, + rows: (&[TestFileRow], &[ScenarioRow]), + settings: &SqlTestSettings, + table_function_arities: HashMap, + ) -> PyResult { + let py: Python<'py> = host.py(); + let contracts: ContractClasses<'py> = self.contract_classes.bind(py).extract()?; + let classes: LoopClasses<'py> = self.loop_classes.bind(py).extract()?; + let context: LoopContext<'_, 'py> = LoopContext { + host, + classes: &classes, + contracts: &contracts, + }; + let validator: Bound<'py, NativeModelValidator> = + self.validator.bind(py).downcast()?.clone(); + let known: KnownResources<'_> = validator.get().known_resources(); + let resolver: &Bound<'py, PyAny> = &classes.external_resolver; + let extended = + |names: &HashSet, method: &str, keyword: &str| -> PyResult> { + let mut names: HashSet = names.clone(); + if !resolver.is_none() { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item(keyword, &names)?; + let added: HashSet = resolver + .call_method(method, (), Some(&arguments))? + .extract()?; + names.extend(added); + } + Ok(names) + }; + let models: HashSet = extended( + known.models, + "extend_sql_test_model_names", + "known_model_names", + )?; + let seeds: HashSet = extended( + known.seeds, + "extend_sql_test_seed_names", + "known_seed_names", + )?; + let sources: HashSet = extended( + known.sources, + "extend_sql_test_source_names", + "known_source_names", + )?; + let scope: TestScope<'_, 'py> = TestScope { + model_loop: self, + context: &context, + settings, + catalog: TargetCatalog { + models, + sources, + seeds, + table_functions: known.table_functions.clone(), + macros: classes.macros.macro_names()?, + }, + functions: known.functions.clone(), + table_function_arities, + }; + let tests: Vec> = scope.attached(rows.0)?; + let scenarios: Vec> = scope.attached_scenarios(rows.1)?; + Ok((tests, scenarios)) + } + + /// Attach every SQL, then Python, function in order, checking table function call arities. + pub(crate) fn run_functions<'py>( + &self, + host: &Bound<'py, PyAny>, + rows: (Vec, Vec), + settings: FunctionSettings, + models: &[(String, Vec)], + ) -> PyResult> { + let py: Python<'py> = host.py(); + let contracts: ContractClasses<'py> = self.contract_classes.bind(py).extract()?; + let classes: LoopClasses<'py> = self.loop_classes.bind(py).extract()?; + let context: LoopContext<'_, 'py> = LoopContext { + host, + classes: &classes, + contracts: &contracts, + }; + let attached: Vec = attached_functions(self, &context, rows, settings)?; + checked_table_function_arities(&context, models, &attached)?; + Ok(attached) + } + + /// Run the model's steps in the attachment's order; `None` hands it to the attachment. + fn finish<'py>( + &self, + py: Python<'py>, + context: &LoopContext<'_, 'py>, + model: LoopModel<'_, 'py>, + ) -> PyResult> { + let LoopModel { + row, + config, + visible, + signatures, + visibility, + } = model; + let ModelLoopRow( + (name, posix_path, relative_path, file_path), + (locations, _, _, _), + authored_query, + contents, + scan, + interpolation, + _, + ) = row; + let values: Bound<'py, PyDict> = config.getattr(VALUES_ATTRIBUTE)?.downcast_into()?; + let schema: Option> = model_schema(context, &values, name)?; + let hooked: bool = hook_presence(&values)?; + let python_hooks: Vec> = if hooked { + signatures.checked(context, &values, &visible.consumer, (name, relative_path))? + } else { + Vec::new() + }; + let schema_usages: Vec> = match &schema { + Some(schema) => { + let declaration = (&SCHEMA_DECLARATION, schema.name.as_str()); + named_usages( + context, + declaration, + &visible.consumer, + (name, relative_path), + )? + } + None => Vec::new(), + }; + let interpolated: String = applied_interpolation(context, interpolation)?; + let (references, stop) = scan.clone().unwrap_or_default(); + let file_keyword: Bound<'py, PyDict> = PyDict::new(py); + file_keyword.set_item( + FILE_PATH_ARGUMENT, + context.classes.path_class.call1((file_path,))?, + )?; + let render_constant: Bound<'py, PyAny> = context.classes.partial.call( + (context.host.getattr(CONSTANT_TEXT_CALLBACK)?,), + Some(&file_keyword), + )?; + let expansion_classes: ExpansionClasses<'py> = ExpansionClasses { + usage_record: context.classes.usage_record.clone(), + compile_error: context.classes.compile_error.clone(), + }; + let declarations: ExpandedDeclarations<'py> = expand_declarations(&DeclarationExpansion { + sql: &interpolated, + references: &references, + stop, + context: &visible.declarations, + consumer: ExpansionConsumer { + identity: Some(visible.consumer.clone()), + relationship_visibility: false, + path: file_path, + }, + render_constant: &render_constant, + classes: &expansion_classes, + })?; + let macros: Option = if calls_macros(&declarations.spliced.sql) { + let model: MacroModel<'_> = MacroModel { + file_path, + relative_path, + contents, + authored_query, + authored: &interpolated == authored_query, + declaration_spans: &declarations.spliced.spans, + }; + Some(expanded_macros( + context, + visible, + &declarations.spliced.sql, + &model, + )?) + } else { + None + }; + let (macro_sql, argument_references, macro_spans): ( + String, + Vec, + Vec, + ) = match ¯os { + Some(macros) => ( + macros.output.sql.clone(), + macros.references.clone(), + macros + .output + .spans + .iter() + .map(|span| span_row(*span)) + .collect(), + ), + None => (declarations.spliced.sql.clone(), Vec::new(), Vec::new()), + }; + let query_sql: String = self + .canonical_query(name, ¯o_sql, &values)? + .map_err(|message| compile_error(context.classes, &message, None))?; + let opt_out_model: OptOutModel<'_, 'py> = OptOutModel { + name, + file_path, + config: &config, + values: &values, + query_sql: &query_sql, + relative_path, + contents, + }; + let (analysed, rejected_opt_out) = self.analysis(context, &opt_out_model)?; + let scanned = match extract_sql_references(&query_sql, &self.syntax) { + ReferenceExtraction::Extracted(scanned) => scanned, + ReferenceExtraction::Failed(failure) => { + let authored: bool = &interpolated == authored_query && query_sql == macro_sql; + let origin: ScanFaultOrigin<'_> = ScanFaultOrigin { + relative_path: posix_path, + contents, + query_sql: authored_query, + passes: authored.then(|| { + vec![ + declarations.spliced.spans.clone(), + spliced_spans(¯o_spans), + ] + }), + }; + let message: String = + located_scan_error(&failure, &query_sql, &origin, &self.syntax); + return Err(compile_error(context.classes, &message, None)); + } + }; + report_invalid_calls( + context, + &self.syntax, + &scanned.invalid_calls, + (Some(name), relative_path, file_path, contents), + )?; + let analysis: (bool, Option>) = ( + analysed && scanned.invalid_calls.is_empty(), + rejected_opt_out, + ); + let references: Vec = merge_call_site_references( + scanned.references.iter().map(reference_row).collect(), + argument_references, + ); + let declared_columns: Option> = + schema.as_ref().map(schema_column_names).transpose()?; + self.validate( + context, + (&config, &values), + (name, &query_sql, relative_path), + (&references, declared_columns), + )?; + let (values, hook_statements, hook_usages) = if hooked { + let hook_model: HookModel<'_, 'py> = HookModel { + name, + posix_path, + relative_path, + file_path, + contents, + authored_query, + visible, + visibility, + }; + let hooks: ExpandedHooks<'py> = expanded_hooks(self, context, &values, &hook_model)?; + (hooks.values, hooks.statements, hooks.usages) + } else { + (values, 0, Vec::new()) + }; + let source: HeaderSource<'_, 'py> = HeaderSource { + name, + values: &values, + locations: locations.bind(py), + relative_path, + visible: (&visible.declarations, &visible.consumer), + schema: schema.as_ref(), + }; + let header: AttachedHeader<'py> = header_schema(py, context, &source)?; + if let Some(message) = legacy_schema_error((name, relative_path), &self.legacy_schema_files) + { + return Err(compile_error(context.classes, &message, None)); + } + let mut named_usages: Vec> = hook_usages; + named_usages.extend(schema_usages); + named_usages.extend(python_hooks); + Ok(FinishedModel { + config, + values, + query_sql, + header, + references, + declarations, + hook_statements, + analysis, + named_usages, + macros: macros.map(|macros| macros.output), + }) + } + + /// Whether the model's SQL is analysed and its rejected opt-out; syntax is checked unless deferred. + fn analysis<'py>( + &self, + context: &LoopContext<'_, 'py>, + model: &OptOutModel<'_, 'py>, + ) -> PyResult<(bool, Option>)> { + let (analysed, rejected) = + match analysis_gate(self.settings, analysis_override(model.values)?) { + AnalysisGate::Conflict => { + return Err(compile_error( + context.classes, + ANALYSIS_CONFLICT, + Some(ANALYSIS_CONFLICT_CODE), + )); + } + AnalysisGate::Enabled => (true, None), + AnalysisGate::Disabled => match rejected_opt_out(self, context, model)? { + OptOut::Accepted => (false, None), + OptOut::Rejected(location) => (true, Some(location)), + }, + }; + if analysed && !self.settings.defer_sql_validation { + reject_unparseable_query(self, context, model)?; + } + Ok((analysed, rejected)) + } + + /// The query with canonical cursor intrinsics, or the message the attachment raises. + fn canonical_query( + &self, + name: &str, + sql: &str, + values: &Bound<'_, PyDict>, + ) -> PyResult> { + let incremental: bool = values.get_item(MATERIALIZED_KEY)?.is_some_and(|value| { + value + .downcast::() + .is_ok_and(|text| text.to_string_lossy() == INCREMENTAL_MATERIALIZATION) + }); + let cursor: Option = match values.get_item(CURSOR_KEY)? { + Some(value) => match value.downcast::() { + Ok(text) => Some(text.to_string_lossy().into_owned()), + Err(_) => None, + }, + None => None, + }; + let model: IntrinsicModel<'_> = IntrinsicModel { + name, + incremental, + cursor: cursor.as_deref(), + }; + Ok(validated_model_intrinsics( + self.python, + sql, + &self.reserved_markers, + &model, + )) + } + + /// Validate the model natively, raising the validator's first error as Python does. + fn validate( + &self, + context: &LoopContext<'_, '_>, + config: (&Bound<'_, PyAny>, &Bound<'_, PyDict>), + model: (&str, &str, &str), + references: (&[ReferenceRow], Option>), + ) -> PyResult<()> { + let (references, declared_columns) = references; + let py: Python<'_> = context.host.py(); + let (config, values) = config; + let external: Vec> = external_errors(context, references, model.2)?; + let rows: Vec<(&str, &str, bool)> = references + .iter() + .zip(&external) + .map(|((kind, name, _, _), error)| (kind.as_str(), name.as_str(), error.is_some())) + .collect(); + let unmanaged: Bound<'_, PyAny> = config + .getattr(RETENTION_ATTRIBUTE)? + .getattr(UNMANAGED_ATTRIBUTE)?; + let declared: Bound<'_, PyAny> = config + .getattr(TABLE_TYPE_ATTRIBUTE)? + .getattr(DECLARED_ATTRIBUTE)?; + let outcome: Bound<'_, PyAny> = self.validator.bind(py).call_method1( + VALIDATE_METHOD, + ( + values.clone(), + model, + (rows, declared_columns, unmanaged, declared), + ), + )?; + validation_outcome(context, &outcome, external, values) + } +} + +/// `_raise_validation_outcome`: nothing, the validator's error, or a rejected reference's error. +pub(crate) fn validation_outcome( + context: &LoopContext<'_, '_>, + outcome: &Bound<'_, PyAny>, + external: Vec>, + values: &Bound<'_, PyDict>, +) -> PyResult<()> { + if outcome.is_none() { + return Ok(()); + } + if let Ok(index) = outcome.extract::() + && let Some(error) = external.into_iter().nth(index).flatten() + { + return Err(error); + } + Err(config_error(context.classes, outcome, Some(values))) +} + +/// `applied_interpolation`: record the interpolation's template reads, then its SQL or error. +fn applied_interpolation( + context: &LoopContext<'_, '_>, + interpolation: &InterpolationRow, +) -> PyResult { + let (sql, reads, error, _) = interpolation; + record_reads(context, reads)?; + match error { + Some(message) => Err(compile_error(context.classes, message, None)), + None => Ok(sql.clone()), + } +} + +/// Record interpolation's environment and context reads for compile reuse. +pub(crate) fn record_reads( + context: &LoopContext<'_, '_>, + reads: &[(String, String)], +) -> PyResult<()> { + if !reads.is_empty() { + let py: Python<'_> = context.host.py(); + context + .classes + .config + .record_template_reads + .call1((PyTuple::new(py, reads)?,))?; + } + Ok(()) +} + +/// `CompileInputError(message, code=...)`, raised as the attachment raises it. +pub(crate) fn compile_error(classes: &LoopClasses<'_>, message: &str, code: Option<&str>) -> PyErr { + let raised = (|| { + let keywords: Bound<'_, PyDict> = PyDict::new(classes.compile_error.py()); + if let Some(code) = code { + keywords.set_item(CODE_ARGUMENT, code)?; + } + classes.compile_error.call((message,), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// `CompileInputError(message, help=help)`. +pub(crate) fn helped_compile_error( + classes: &LoopClasses<'_>, + message: &str, + help: Option<&str>, +) -> PyErr { + let raised = (|| { + let keywords: Bound<'_, PyDict> = PyDict::new(classes.compile_error.py()); + keywords.set_item(HELP_ARGUMENT, help)?; + classes.compile_error.call((message,), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// The Python exception a native config error raises as, given the values it names. +pub(crate) fn config_error( + classes: &LoopClasses<'_>, + error: &Bound<'_, PyAny>, + values: Option<&Bound<'_, PyDict>>, +) -> PyErr { + let raised = (|| { + let keywords: Bound<'_, PyDict> = PyDict::new(error.py()); + keywords.set_item(ERROR_ARGUMENT, error)?; + keywords.set_item(VALUES_ARGUMENT, values)?; + classes.config_error.call((), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} + +/// Span rows as the spliced spans the scan-fault locator maps through. +fn spliced_spans(rows: &[SpanRow]) -> Vec { + rows.iter() + .map( + |&(source_start, source_end, output_start, output_end)| SplicedSpan { + source_start, + source_end, + output_start, + output_end, + }, + ) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs new file mode 100644 index 0000000000..f80603a5f8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs @@ -0,0 +1,158 @@ +//! `report_invalid_reference_calls`: P012 for each rejected reference call, located when authored. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_analysis::compiled_project::main::located_scan_error::located_scan_error; +use sqlbuild_analysis::compiled_project::main::text_position::text_position; +use sqlbuild_analysis::compiled_project::models::ScanFaultOrigin; +use sqlbuild_analysis::compiled_project::models::TextPosition; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::{ + InvalidReferenceCall, ReferenceExtraction, SqlReference, +}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compile_error}; + +const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; +const MODEL_RESOURCE: &str = "model"; + +/// `(model name or none for a function, relative path, file path, contents)` of a file. +pub(crate) type CallOrigin<'a> = (Option<&'a str>, &'a str, &'a str, &'a str); + +/// Report each rejected call in order, at its authored location when the authored text has it. +pub(crate) fn report_invalid_calls( + context: &LoopContext<'_, '_>, + syntax: &LexicalSyntax, + invalid_calls: &[InvalidReferenceCall], + origin: CallOrigin<'_>, +) -> PyResult<()> { + if invalid_calls.is_empty() { + return Ok(()); + } + let (_, _, _, contents) = origin; + let authored: HashMap> = match extract_sql_references(contents, syntax) { + ReferenceExtraction::Extracted(scan) => { + let mut starts: HashMap> = HashMap::new(); + for call in scan.invalid_calls { + starts.entry(call.call).or_default().push(call.start); + } + starts + } + ReferenceExtraction::Failed(_) => HashMap::new(), + }; + let mut seen: HashMap<&str, usize> = HashMap::new(); + for invalid_call in invalid_calls { + let occurrence: &mut usize = seen.entry(invalid_call.call.as_str()).or_default(); + let start: Option = authored + .get(&invalid_call.call) + .and_then(|starts| starts.get(*occurrence).copied()); + *occurrence += 1; + report_invalid_call(context, invalid_call, start, origin)?; + } + Ok(()) +} + +/// Report one rejected call as P012, at `start` in the origin's contents when given. +pub(crate) fn report_invalid_call( + context: &LoopContext<'_, '_>, + invalid_call: &InvalidReferenceCall, + start: Option, + origin: CallOrigin<'_>, +) -> PyResult<()> { + let (name, relative_path, file_path, contents) = origin; + let py: Python<'_> = context.host.py(); + let path: Bound<'_, PyAny> = context.classes.path_class.call1((relative_path,))?; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", "error")?; + fields.set_item("code", REFERENCE_CALL_SYNTAX_CODE)?; + fields.set_item("message", &invalid_call.message)?; + fields.set_item("resource_type", name.map(|_| MODEL_RESOURCE))?; + fields.set_item("resource_name", name)?; + fields.set_item("help", &invalid_call.help)?; + let key: Bound<'_, PyTuple> = match start { + Some(start) => { + let begin: TextPosition = text_position(contents, start); + let end: TextPosition = + text_position(contents, start + invalid_call.call.chars().count()); + let location: Bound<'_, PyDict> = PyDict::new(py); + location.set_item("path", &path)?; + location.set_item("line", begin.line)?; + location.set_item("column", begin.column)?; + location.set_item("end_line", end.line)?; + location.set_item("end_column", end.column)?; + fields.set_item( + "location", + context + .contracts + .source_location + .call((), Some(&location))?, + )?; + PyTuple::new( + py, + [ + REFERENCE_CALL_SYNTAX_CODE.to_owned(), + file_path.to_owned(), + format!("{:09}:{:09}", begin.line, begin.column), + invalid_call.call.clone(), + ], + )? + } + None => { + fields.set_item("path", &path)?; + PyTuple::new( + py, + [ + REFERENCE_CALL_SYNTAX_CODE.to_owned(), + file_path.to_owned(), + invalid_call.call.clone(), + ], + )? + } + }; + let diagnostic: Bound<'_, PyAny> = context + .classes + .compiler_diagnostic + .call((), Some(&fields))?; + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("key", key)?; + arguments.set_item("diagnostic", diagnostic)?; + context + .classes + .report_diagnostic + .call((), Some(&arguments))?; + Ok(()) +} + +/// `extract_sql_references(sql, origin=file)`, reporting rejected calls and located scan errors. +pub(crate) fn origin_references( + context: &LoopContext<'_, '_>, + syntax: &LexicalSyntax, + sql: &str, + file: (&str, &str, &str), +) -> PyResult> { + let (file_path, relative_path, contents) = file; + let scanned = match extract_sql_references(sql, syntax) { + ReferenceExtraction::Extracted(scanned) => scanned, + ReferenceExtraction::Failed(failure) => { + let origin: ScanFaultOrigin<'_> = ScanFaultOrigin { + relative_path, + contents, + query_sql: sql, + passes: None, + }; + let message: String = located_scan_error(&failure, sql, &origin, syntax); + return Err(compile_error(context.classes, &message, None)); + } + }; + report_invalid_calls( + context, + syntax, + &scanned.invalid_calls, + (None, relative_path, file_path, contents), + )?; + Ok(scanned.references) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs new file mode 100644 index 0000000000..2d0523e827 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs @@ -0,0 +1,313 @@ +//! A model's effective config: its path default, the native build, and reuse across models. + +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::{FromPyObject, IntoPyObject}; +use sqlbuild_model_config::config_presence::main::blocks_config_reuse::blocks_config_reuse; +use sqlbuild_model_config::config_presence::main::contains_template::contains_template; +use sqlbuild_model_config::config_presence::main::mutable_config_value::mutable_config_value; +use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; + +use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, config_error}; +use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::model_config_builder::{ + BuiltConfig, ConfigBuild, NativeModelConfigBuilder, +}; + +const COLUMNS_KEY: &str = "columns"; +const DESCRIPTION_KEY: &str = "description"; +const MATERIALIZED_KEY: &str = "materialized"; +const VALUES_FIELD: &str = "values"; +const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +const PERMANENT_TABLE_TYPE: &str = "permanent"; +const TRANSIENT_TABLE_TYPE: &str = "transient"; + +/// The builder, storage defaults and Python classes a model's `CompileModelConfig` is made of. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct ConfigClasses<'py> { + pub(crate) builder: Bound<'py, NativeModelConfigBuilder>, + /// Inherited `(retention, table type)` by table-backed materialization, or `None`. + inherited_storage: Bound<'py, PyDict>, + table_backed: Vec, + compile_model_config: Bound<'py, PyAny>, + resolved_retention: Bound<'py, PyAny>, + retention_source: Bound<'py, PyAny>, + resolved_table_type: Bound<'py, PyAny>, + table_type: Bound<'py, PyAny>, + table_type_source: Bound<'py, PyAny>, + /// `record_template_reads`, the compile-reuse sink of template reads. + pub(crate) record_template_reads: Bound<'py, PyAny>, +} + +/// What a path default's models inherit, and whether a model may reuse another's config. +struct InheritedLayer<'py> { + reusable: bool, + values: Bound<'py, PyDict>, +} + +/// A reusable config: its values without header metadata, and its other fields. +struct ReusedConfig<'py> { + values: Bound<'py, PyDict>, + fields: Bound<'py, PyDict>, +} + +/// `_ReusableModelConfigCache` and the builds the loop ran, for one compile. +#[derive(Default)] +pub(crate) struct ModelConfigs<'py> { + layers: RefCell, InheritedLayer<'py>>>, + reused: RefCell, ReusedConfig<'py>>>, + builds: Cell, +} + +impl<'py> ModelConfigs<'py> { + /// `_effective_model_config`: the reused or natively built config of one model. + pub(crate) fn effective( + &self, + context: &LoopContext<'_, 'py>, + header: &Bound<'py, PyDict>, + model: (&str, &str), + ) -> PyResult> { + let (name, relative_path) = model; + let py: Python<'py> = header.py(); + let classes: &ConfigClasses<'py> = &context.classes.config; + let builder: &NativeModelConfigBuilder = classes.builder.get(); + let matched: Option = builder + .selected_path_default(py, relative_path)? + .map_err(|error| config_error(context.classes, error.bind(py).as_any(), None))?; + let metadata: Option> = reusable_metadata(header)?; + let reusable: bool = match metadata { + Some(_) => self.reusable(context, &matched)?, + None => false, + }; + if reusable + && let Some(metadata) = &metadata + && let Some(cached) = self.reused.borrow().get(&matched) + { + let values: Bound<'py, PyDict> = cached.values.copy()?; + values.update(metadata.as_mapping())?; + return view(classes, &values, &cached.fields); + } + let built: ConfigBuild<'py> = builder.built(py, header, matched.as_deref(), name)?; + self.builds.set(self.builds.get() + 1); + let built: BuiltConfig<'py> = match built { + ConfigBuild::Built(built) => built, + ConfigBuild::Rejected(error) => { + return Err(config_error(context.classes, error.bind(py).as_any(), None)); + } + }; + let fields: Bound<'py, PyDict> = config_fields(classes, &built, matched.as_deref())?; + if reusable && let Some(metadata) = &metadata { + self.remember(&matched, metadata, &built.values, &fields)?; + } + view(classes, &built.values, &fields) + } + + /// The number of native config builds the loop ran. + pub(crate) fn builds(&self) -> usize { + self.builds.get() + } + + /// `_is_reusable`: no templates in the inherited values or the target namespace, nothing mutable. + fn reusable(&self, context: &LoopContext<'_, 'py>, matched: &Option) -> PyResult { + if let Some(layer) = self.layers.borrow().get(matched) { + return Ok(layer.reusable); + } + let py: Python<'py> = context.host.py(); + let builder: &NativeModelConfigBuilder = context.classes.config.builder.get(); + let values: Bound<'py, PyDict> = builder + .inherited_values(py, matched.as_deref())? + .map_err(|error| config_error(context.classes, error.bind(py).as_any(), None))?; + let checked: Bound<'py, PyTuple> = PyTuple::new( + py, + [values.as_any(), builder.target_namespace(py)?.as_any()], + )?; + let mut mutable: bool = false; + for key in HOOK_KEYS { + mutable |= values.contains(key)?; + } + for (_, value) in values.iter() { + mutable |= mutable_config_value(&PyNode(value)); + } + let reusable: bool = !contains_template(&PyNode(checked.into_any())) && !mutable; + self.layers + .borrow_mut() + .insert(matched.clone(), InheritedLayer { reusable, values }); + Ok(reusable) + } + + /// `remember`: the built values with header metadata replaced by what the model inherits. + fn remember( + &self, + matched: &Option, + metadata: &Bound<'py, PyDict>, + values: &Bound<'py, PyDict>, + fields: &Bound<'py, PyDict>, + ) -> PyResult<()> { + let py: Python<'py> = values.py(); + let layers = self.layers.borrow(); + let Some(inherited) = layers.get(matched) else { + return Ok(()); + }; + let reusable_values: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in values.iter() { + if !metadata.contains(&key)? { + reusable_values.set_item(key, value)?; + } + } + for key in metadata.keys() { + if let Some(value) = inherited.values.get_item(&key)? { + reusable_values.set_item(key, value)?; + } + } + self.reused.borrow_mut().insert( + matched.clone(), + ReusedConfig { + values: reusable_values, + fields: fields.clone(), + }, + ); + Ok(()) + } +} + +/// `_reusable_metadata`: the header when only plain columns and description vary, else `None`. +fn reusable_metadata<'py>(header: &Bound<'py, PyDict>) -> PyResult>> { + let py: Python<'py> = header.py(); + if header.is_empty() { + return Ok(Some(PyDict::new(py))); + } + for key in header.keys() { + let reusable_key: bool = key.downcast::().is_ok_and(|key| { + key.to_str() + .is_ok_and(|key| key == COLUMNS_KEY || key == DESCRIPTION_KEY) + }); + if !reusable_key { + return Ok(None); + } + } + for key in [DESCRIPTION_KEY, COLUMNS_KEY] { + if let Some(value) = header.get_item(key)? + && blocks_config_reuse(&PyNode(value)) + { + return Ok(None); + } + } + Ok(Some(header.clone())) +} + +/// The model's `CompileModelConfig`, once its template reads are recorded. +pub(crate) fn built_config<'py>( + classes: &ConfigClasses<'py>, + built: &BuiltConfig<'py>, + matched: Option<&str>, +) -> PyResult> { + let fields: Bound<'py, PyDict> = config_fields(classes, built, matched)?; + view(classes, &built.values, &fields) +} + +/// Every `CompileModelConfig` field but `values`, recording the build's template reads first. +fn config_fields<'py>( + classes: &ConfigClasses<'py>, + built: &BuiltConfig<'py>, + matched: Option<&str>, +) -> PyResult> { + let py: Python<'py> = built.values.py(); + if !built.reads.is_empty() { + classes + .record_template_reads + .call1((PyTuple::new(py, &built.reads)?,))?; + } + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let (inherited_retention, inherited_table_type): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + classes + .inherited_storage + .as_any() + .get_item(storage_key(classes, &built.values)?)? + .extract()?; + let retention: Bound<'py, PyAny> = match built.retention { + RetentionOverride::Inherit => inherited_retention, + RetentionOverride::Unmanaged => resolved_retention(classes, none.clone(), true)?, + RetentionOverride::Days(days) => { + resolved_retention(classes, days.into_pyobject(py)?.into_any(), false)? + } + }; + let table_type: Bound<'py, PyAny> = match built.table_type { + TableTypeOverride::Inherit => inherited_table_type, + TableTypeOverride::Permanent => resolved_table_type(classes, PERMANENT_TABLE_TYPE)?, + TableTypeOverride::Transient => resolved_table_type(classes, TRANSIENT_TABLE_TYPE)?, + }; + let logical_schema: Bound<'py, PyAny> = built.logical_schema.clone().unwrap_or(none.clone()); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("model_header_keys", PyTuple::new(py, &built.header_keys)?)?; + fields.set_item("matched_path_default", matched)?; + fields.set_item("logical_schema", &logical_schema)?; + fields.set_item( + "layer_schema", + if built.layer_schema_configured { + logical_schema + } else { + none.clone() + }, + )?; + fields.set_item( + "logical_database", + built.logical_database.clone().unwrap_or(none), + )?; + fields.set_item("time_travel_retention", retention)?; + fields.set_item("table_type", table_type)?; + Ok(fields) +} + +/// The materialization whose inherited storage policy applies, or `None` when not table-backed. +fn storage_key<'py>( + classes: &ConfigClasses<'py>, + values: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = values.py(); + if let Some(materialized) = values.get_item(MATERIALIZED_KEY)? + && let Ok(text) = materialized.downcast::() + && let Ok(text) = text.to_str() + && classes.table_backed.iter().any(|name| name == text) + { + return Ok(materialized); + } + Ok(py.None().into_bound(py)) +} + +fn resolved_retention<'py>( + classes: &ConfigClasses<'py>, + desired_days: Bound<'py, PyAny>, + unmanaged: bool, +) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(desired_days.py()); + fields.set_item("desired_days", desired_days)?; + fields.set_item("unmanaged", unmanaged)?; + fields.set_item("source", &classes.retention_source)?; + classes.resolved_retention.call((), Some(&fields)) +} + +fn resolved_table_type<'py>( + classes: &ConfigClasses<'py>, + value: &str, +) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(classes.table_type.py()); + fields.set_item("value", classes.table_type.call1((value,))?)?; + fields.set_item("source", &classes.table_type_source)?; + fields.set_item("declared", true)?; + classes.resolved_table_type.call((), Some(&fields)) +} + +/// A `CompileModelConfig` of `values` and the other `fields`. +fn view<'py>( + classes: &ConfigClasses<'py>, + values: &Bound<'py, PyDict>, + fields: &Bound<'py, PyDict>, +) -> PyResult> { + let arguments: Bound<'py, PyDict> = fields.copy()?; + arguments.set_item(VALUES_FIELD, values)?; + classes.compile_model_config.call((), Some(&arguments)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs new file mode 100644 index 0000000000..fbc622549d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs @@ -0,0 +1,342 @@ +//! Header contract objects the model loop builds: factory audits, dynamic families, enum columns. + +use pyo3::BoundObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +use crate::bindings::_helpers::compiled_project::model_loop::LoopContext; +use crate::bindings::_helpers::compiled_project::project::{ + DynamicFamilyPy, SchemaColumnPy, SchemaPy, +}; + +const CASES_ATTRIBUTE: &str = "cases"; +const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; +const LINE_ATTRIBUTE: &str = "line"; +const DEFINITION_ATTRIBUTE: &str = "definition"; +const DEFINITION_NAME_FIELD: &str = "definition_name"; +const ARGUMENTS_FIELD: &str = "arguments"; +const NAME_FIELD: &str = "name"; +const TYPE_FIELD: &str = "type"; +const NULLABLE_FIELD: &str = "nullable"; +const AUDITS_FIELD: &str = "audits"; +const MEMBERS_FIELD: &str = "members"; +const VALUE_FIELD: &str = "value"; +const VALUES_ARGUMENT: &str = "values"; +const SCALAR_TYPE_ATTRIBUTE: &str = "scalar_type"; +const ENUM_VISIBILITY_ATTRIBUTE: &str = "enum_visibility"; +const DECLARATION_FIELD: &str = "declaration"; +const THROUGH_FIELD: &str = "through"; +const CONSUMER_FIELD: &str = "consumer"; +const KIND_FIELD: &str = "kind"; +const PATH_FIELD: &str = "path"; +const COLUMN_FIELD: &str = "column"; +const LOCATION_FIELD: &str = "location"; +const CASEFOLD_METHOD: &str = "casefold"; +const ACCEPTED_VALUES_AUDIT: &str = "accepted_values"; +const CONTRACT_KEY: &str = "contract"; +/// `AuditCase` fields copied onto each generated `SchemaAuditInstance` under the same name. +const COPIED_CASE_FIELDS: [&str; 8] = [ + "name", + "severity", + "run_scope", + "always_run", + "description", + "thresholds", + "minimum_samples", + "evidence_limit", +]; +/// `SchemaColumn` fields `dataclasses.replace` carries over unchanged. +const KEPT_COLUMN_FIELDS: [&str; 6] = [ + "name", + "nullable", + "description", + "meta", + "location", + "migrate_from", +]; +const PIVOT_COLUMN_FIELD: &str = "pivot_column"; +const VALUE_COLUMN_FIELD: &str = "value_column"; +const AGGREGATE_FIELD: &str = "aggregate"; +const NAME_PATTERN_FIELD: &str = "name_pattern"; + +/// A model's enum-typed columns resolved to scalar types, and the usages they generate. +pub(crate) struct EnumColumns<'py> { + pub(crate) columns: Bound<'py, PyTuple>, + /// Column name to its enum declaration, as `CompileModelInput.enum_columns` holds them. + pub(crate) declarations: Bound<'py, PyDict>, + pub(crate) usages: Vec>, +} + +/// Each referenced factory's cases as audit instances, with their identities. +pub(crate) fn factory_audits<'py>( + py: Python<'py>, + context: &LoopContext<'_, 'py>, + references: &[String], +) -> PyResult<(Vec>, Vec)> { + let mut audits: Vec> = Vec::new(); + let mut identities: Vec = Vec::new(); + for reference in references { + let Some(factory) = context.classes.audit_factories.get_item(reference)? else { + continue; + }; + let location: Bound<'py, PyAny> = keywords( + &context.contracts.source_location, + &[ + (PATH_FIELD, factory.getattr(RELATIVE_PATH_ATTRIBUTE)?), + (LINE_ATTRIBUTE, factory.getattr(LINE_ATTRIBUTE)?), + (COLUMN_FIELD, 1_u8.into_pyobject_any(py)?), + ], + )?; + for case in factory.getattr(CASES_ATTRIBUTE)?.try_iter()? { + let case: Bound<'py, PyAny> = case?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(DEFINITION_NAME_FIELD, case.getattr(DEFINITION_ATTRIBUTE)?)?; + fields.set_item( + ARGUMENTS_FIELD, + py.get_type::() + .call1((case.getattr(ARGUMENTS_FIELD)?,))?, + )?; + for field in COPIED_CASE_FIELDS { + fields.set_item(field, case.getattr(field)?)?; + } + fields.set_item(LOCATION_FIELD, &location)?; + let audit: Bound<'py, PyAny> = context + .contracts + .schema_audit_instance + .call((), Some(&fields))?; + identities.push(audit_identity(&audit)?); + audits.push(audit); + } + } + Ok((audits, identities)) +} + +/// `audit.name or audit.definition_name`. +pub(crate) fn audit_identity(audit: &Bound<'_, PyAny>) -> PyResult { + let name: Bound<'_, PyAny> = audit.getattr(NAME_FIELD)?; + if name.is_truthy()? { + return name.extract(); + } + audit.getattr(DEFINITION_NAME_FIELD)?.extract() +} + +/// The parsed families as `SchemaDynamicColumnFamily` objects. +pub(crate) fn dynamic_family_objects<'py>( + py: Python<'py>, + context: &LoopContext<'_, 'py>, + families: &[DynamicFamily], +) -> PyResult>> { + families + .iter() + .map(|family| { + keywords( + &context.contracts.schema_dynamic_column_family, + &[ + (NAME_FIELD, family.name.as_str().into_pyobject_any(py)?), + ( + PIVOT_COLUMN_FIELD, + family.pivot_column.as_str().into_pyobject_any(py)?, + ), + ( + VALUE_COLUMN_FIELD, + family.value_column.as_str().into_pyobject_any(py)?, + ), + ( + AGGREGATE_FIELD, + family.aggregate.as_str().into_pyobject_any(py)?, + ), + (TYPE_FIELD, family.data_type.as_str().into_pyobject_any(py)?), + ( + NAME_PATTERN_FIELD, + family.name_pattern.as_deref().into_pyobject_any(py)?, + ), + ], + ) + }) + .collect() +} + +/// `resolve_enum_contract_columns` and `_generated_enum_usages` for a header schema entry. +pub(crate) fn enum_columns<'py>( + context: &LoopContext<'_, 'py>, + visible: ( + &Bound<'py, PyDict>, + (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + ), + values: &Bound<'py, PyDict>, + columns: &Bound<'py, PyTuple>, +) -> PyResult> { + let py: Python<'py> = columns.py(); + let (enums, (declarations, consumer)) = visible; + let enforced: bool = match values.get_item(CONTRACT_KEY)? { + Some(value) => value.eq(&context.classes.contract_enforced)?, + None => false, + }; + let resolved: Bound<'py, PyDict> = PyDict::new(py); + let mut resolved_columns: Vec> = Vec::with_capacity(columns.len()); + for column in columns.iter() { + let column_type: Bound<'py, PyAny> = column.getattr(TYPE_FIELD)?; + let key: Bound<'py, PyAny> = if column_type.is_truthy()? { + column_type + } else { + "".into_pyobject_any(py)? + }; + let Some(declaration) = enums.get_item(&key)? else { + resolved_columns.push(column); + continue; + }; + resolved.set_item(column.getattr(NAME_FIELD)?, &declaration)?; + let mut audits: Vec> = column + .getattr(AUDITS_FIELD)? + .try_iter()? + .collect::>()?; + if enforced { + let generated: Bound<'py, PyAny> = accepted_values(context, &declaration)?; + let mut present: bool = false; + for audit in &audits { + if audit.eq(&generated)? { + present = true; + break; + } + } + if !present { + audits.push(generated); + } + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in KEPT_COLUMN_FIELDS { + fields.set_item(field, column.getattr(field)?)?; + } + fields.set_item(TYPE_FIELD, declaration.getattr(SCALAR_TYPE_ATTRIBUTE)?)?; + fields.set_item(AUDITS_FIELD, PyTuple::new(py, audits)?)?; + resolved_columns.push(context.contracts.schema_column.call((), Some(&fields))?); + } + let usages: Vec> = + generated_usages(context, declarations, consumer, &resolved)?; + Ok(EnumColumns { + columns: PyTuple::new(py, resolved_columns)?, + declarations: resolved, + usages, + }) +} + +/// The schema row the retained facts are read from, as `_schema_row` builds it. +pub(crate) fn schema_row( + columns: &Bound<'_, PyTuple>, + (audit_count, model_path): (usize, &Bound<'_, PyAny>), + families: &[Bound<'_, PyAny>], + switches: (bool, bool), +) -> PyResult { + let mut column_rows: Vec = Vec::with_capacity(columns.len()); + for column in columns.iter() { + let location: Bound<'_, PyAny> = column.getattr(LOCATION_FIELD)?; + let declared_outside: bool = + !location.is_none() && location.getattr(PATH_FIELD)?.ne(model_path)?; + column_rows.push(( + column.getattr(NAME_FIELD)?.extract()?, + column.getattr(TYPE_FIELD)?.extract()?, + column.getattr(NULLABLE_FIELD)?.extract()?, + u32::try_from(column.getattr(AUDITS_FIELD)?.len()?).unwrap_or(u32::MAX), + declared_outside, + )); + } + let mut family_rows: Vec = Vec::with_capacity(families.len()); + for family in families { + let name: Bound<'_, PyAny> = family.getattr(NAME_FIELD)?; + family_rows.push(( + name.call_method0(CASEFOLD_METHOD)?.extract()?, + name.extract()?, + family.getattr(PIVOT_COLUMN_FIELD)?.extract()?, + family.getattr(VALUE_COLUMN_FIELD)?.extract()?, + family.getattr(AGGREGATE_FIELD)?.extract()?, + family.getattr(TYPE_FIELD)?.extract()?, + family.getattr(NAME_PATTERN_FIELD)?.extract()?, + )); + } + Ok(( + u32::try_from(audit_count).unwrap_or(u32::MAX), + column_rows, + family_rows, + switches, + )) +} + +fn accepted_values<'py>( + context: &LoopContext<'_, 'py>, + declaration: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = declaration.py(); + let mut members: Vec> = Vec::new(); + for member in declaration.getattr(MEMBERS_FIELD)?.try_iter()? { + members.push(member?.getattr(VALUE_FIELD)?); + } + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item(VALUES_ARGUMENT, PyTuple::new(py, members)?)?; + keywords( + &context.contracts.schema_audit_instance, + &[ + ( + DEFINITION_NAME_FIELD, + ACCEPTED_VALUES_AUDIT.into_pyobject_any(py)?, + ), + (ARGUMENTS_FIELD, arguments.into_any()), + ], + ) +} + +fn generated_usages<'py>( + context: &LoopContext<'_, 'py>, + declarations: &Bound<'py, PyAny>, + consumer: &Bound<'py, PyAny>, + resolved: &Bound<'py, PyDict>, +) -> PyResult>> { + let visibility: Bound<'py, PyDict> = declarations + .getattr(ENUM_VISIBILITY_ATTRIBUTE)? + .downcast_into()?; + let mut usages: Vec> = Vec::new(); + for declaration in resolved.values() { + let Some(records) = visibility.get_item(declaration.getattr(NAME_FIELD)?)? else { + continue; + }; + for record in records.try_iter()? { + let record: Bound<'py, PyAny> = record?; + usages.push(keywords( + &context.classes.usage_record, + &[ + (CONSUMER_FIELD, consumer.clone()), + (DECLARATION_FIELD, record.getattr(DECLARATION_FIELD)?), + (KIND_FIELD, context.classes.generated_usage.clone()), + (THROUGH_FIELD, record.getattr(THROUGH_FIELD)?), + ], + )?); + } + } + Ok(usages) +} + +fn keywords<'py>( + class: &Bound<'py, PyAny>, + fields: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(class.py()); + for (key, value) in fields { + arguments.set_item(*key, value)?; + } + class.call((), Some(&arguments)) +} + +/// `into_pyobject` for any value, as a `PyAny`. +trait IntoPyAny<'py> { + fn into_pyobject_any(self, py: Python<'py>) -> PyResult>; +} + +impl<'py, T> IntoPyAny<'py> for T +where + T: pyo3::IntoPyObject<'py>, + PyErr: From, +{ + fn into_pyobject_any(self, py: Python<'py>) -> PyResult> { + Ok(self.into_pyobject(py)?.into_any().into_bound()) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs new file mode 100644 index 0000000000..d968d3fae7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs @@ -0,0 +1,70 @@ +//! `__dbt_ref` references, checked by the external resolver that owns the dbt manifest. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::PyDict; + +use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compile_error}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; + +const DBT_REF_KIND: &str = "dbt_ref"; +const VALIDATE_METHOD: &str = "validate_reference"; +const MISSING_MANIFEST_CODE: &str = "C214"; +const MISSING_MANIFEST_HELP: &str = + "Run dbt compile or configure dbt target_path so SQLBuild can read manifest.json."; +const QUOTE: char = '"'; + +/// `_external_reference_error` per reference, in order: the resolver's error, or `None`. +pub(crate) fn external_errors( + context: &LoopContext<'_, '_>, + references: &[ReferenceRow], + relative_path: &str, +) -> PyResult>> { + let mut errors: Vec> = Vec::with_capacity(references.len()); + for (kind, name, package, _) in references { + errors.push(if kind == DBT_REF_KIND { + external_error(context, (kind, name, package.as_deref()), relative_path)? + } else { + None + }); + } + Ok(errors) +} + +fn external_error( + context: &LoopContext<'_, '_>, + reference: (&str, &str, Option<&str>), + relative_path: &str, +) -> PyResult> { + let (kind, name, package) = reference; + let resolver: &Bound<'_, PyAny> = &context.classes.external_resolver; + let py: Python<'_> = resolver.py(); + if resolver.is_none() { + let quoted: String = name.replace(QUOTE, "\"\""); + let message: String = format!( + "Model file {relative_path} uses __{kind}({QUOTE}{quoted}{QUOTE}) but no dbt \ + manifest was found" + ); + let error: PyErr = compile_error(context.classes, &message, Some(MISSING_MANIFEST_CODE)); + error.value(py).setattr("help", MISSING_MANIFEST_HELP)?; + return Ok(Some(error)); + } + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("ref_kind", context.classes.reference_kind.call1((kind,))?)?; + arguments.set_item("ref_name", name)?; + arguments.set_item("ref_package", package)?; + arguments.set_item( + "owner_relative_sql_path", + context.classes.path_class.call1((relative_path,))?, + )?; + match resolver.call_method(VALIDATE_METHOD, (), Some(&arguments)) { + Ok(_) => Ok(None), + Err(error) + if error + .value(py) + .is_instance(&context.classes.compile_error)? => + { + Ok(Some(error)) + } + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs new file mode 100644 index 0000000000..b21ececf28 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -0,0 +1,221 @@ +//! A model's MODEL header attached as its schema entry, by the native header parser. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods}; +use sqlbuild_analysis::compiled_project::models::ModelSchemaFact; +use sqlbuild_model_config::header_metadata::main::audit_factory_references::audit_factory_references; +use sqlbuild_model_config::header_metadata::main::merged_audit_identities::merged_audit_identities; +use sqlbuild_model_config::header_metadata::main::parse_dynamic_columns::parse_dynamic_columns; +use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_model_config::model_validation::models::ColumnMigration; + +use crate::bindings::_helpers::compiled_project::model_loop::{ + LoopContext, compile_error, config_error, +}; +use crate::bindings::_helpers::compiled_project::model_loop_contracts::{ + EnumColumns, audit_identity, dynamic_family_objects, enum_columns, factory_audits, schema_row, +}; +use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; +use crate::bindings::_helpers::compiled_project::model_loop_schema::{ + ModelSchema, merged_columns, schema_enums, +}; +use crate::bindings::_helpers::compiled_project::project::schema_fact; +use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; +use crate::bindings::_helpers::model_config::config_errors::native_config_error; +use crate::bindings::_helpers::model_config::header_metadata::{audit_tuple, column_tuple}; + +const DESCRIPTION_KEY: &str = "description"; +const COLUMNS_KEY: &str = "columns"; +const AUDITS_KEY: &str = "audits"; +const DYNAMIC_COLUMNS_KEY: &str = "dynamic_columns"; +const AUDIT_FACTORIES_KEY: &str = "audit_factories"; +const STRIP_METHOD: &str = "strip"; +const NAME_FIELD: &str = "name"; +const TYPE_FIELD: &str = "type"; +const MIGRATE_FROM_FIELD: &str = "migrate_from"; +const ENUMS_FIELD: &str = "enums"; + +/// What a model's header schema entry is attached from. +pub(crate) struct HeaderSource<'a, 'py> { + pub(crate) name: &'a str, + pub(crate) values: &'a Bound<'py, PyDict>, + pub(crate) locations: &'a Bound<'py, PyDict>, + pub(crate) relative_path: &'a str, + /// The model's visible declarations and the identity its usages are recorded for. + pub(crate) visible: (&'a Bound<'py, PyAny>, &'a Bound<'py, PyAny>), + pub(crate) schema: Option<&'a ModelSchema<'py>>, +} + +/// The header's description and, when it declares any schema metadata, its entry. +pub(crate) struct AttachedHeader<'py> { + pub(crate) description: Option, + pub(crate) schema: Option>, +} + +/// A header schema entry's Python contract objects, enum columns and retained fact. +pub(crate) struct HeaderSchema<'py> { + pub(crate) columns: Bound<'py, PyTuple>, + pub(crate) audits: Bound<'py, PyTuple>, + pub(crate) dynamic_columns: Bound<'py, PyTuple>, + pub(crate) type_enforcement: bool, + pub(crate) enum_columns: Bound<'py, PyDict>, + pub(crate) generated_usages: Vec>, + pub(crate) model_schema: Option, + pub(crate) fact: ModelSchemaFact, +} + +/// The attached header entry, raising the attachment's errors in its order. +pub(crate) fn header_schema<'py>( + py: Python<'py>, + context: &LoopContext<'_, 'py>, + source: &HeaderSource<'_, 'py>, +) -> PyResult> { + let description: Option = match (description(context, source)?, source.schema) { + (None, Some(schema)) => schema.declaration.getattr(DESCRIPTION_KEY)?.extract()?, + (description, _) => description, + }; + let columns: Option> = present(source.values, COLUMNS_KEY)?; + let audits: Option> = present(source.values, AUDITS_KEY)?; + let dynamic: Option> = present(source.values, DYNAMIC_COLUMNS_KEY)?; + let factories: Option> = present(source.values, AUDIT_FACTORIES_KEY)?; + if source.schema.is_none() + && description.is_none() + && columns.is_none() + && audits.is_none() + && dynamic.is_none() + && factories.is_none() + { + return Ok(AttachedHeader { + description, + schema: None, + }); + } + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let node = |value: Option>| PyNode(value.unwrap_or_else(|| none.clone())); + let model: (&str, &str) = (source.name, source.relative_path); + let metadata = parse_header_metadata(&node(columns), &node(audits), source.relative_path); + let parsed_columns = metadata + .columns + .map_err(|error| raised_config_error(py, context, error))?; + let local_columns: Bound<'py, PyTuple> = + column_tuple(py, context.contracts, &parsed_columns, source.locations)?; + let columns: Bound<'py, PyTuple> = match source.schema { + Some(schema) => merged_columns(context, schema, &local_columns, model)?, + None => local_columns, + }; + let families = dynamic_families(py, context, &node(dynamic), model)?; + let parsed_audits = metadata + .audits + .map_err(|error| raised_config_error(py, context, error))?; + let available: Vec = context.classes.audit_factories.keys().extract()?; + let references: Vec = audit_factory_references(&node(factories), model, &available) + .map_err(|error| raised_config_error(py, context, error))?; + let direct_audits: Bound<'py, PyTuple> = audit_tuple(py, context.contracts, &parsed_audits)?; + let (generated, generated_identities) = factory_audits(py, context, &references)?; + let direct_identities: Vec = direct_audits + .iter() + .map(|audit| audit_identity(&audit)) + .collect::>()?; + merged_audit_identities(&direct_identities, &generated_identities, model) + .map_err(|error| raised_config_error(py, context, error))?; + let migrations: Vec = columns + .iter() + .map(|column| column_migration(&column)) + .collect::>()?; + if let Some(error) = column_migration_error(source.name, source.values, &migrations)? { + let error = native_config_error(py, error)?.into_bound(py).into_any(); + return Err(config_error(context.classes, &error, Some(source.values))); + } + let mut type_enforcement: bool = !families.is_empty(); + for column in columns.iter() { + type_enforcement |= !column.getattr(TYPE_FIELD)?.is_none(); + } + let audits: Bound<'py, PyTuple> = PyTuple::new( + py, + direct_audits.iter().chain(generated).collect::>(), + )?; + let families: Vec> = dynamic_family_objects(py, context, &families)?; + let enums: Bound<'py, PyDict> = source.visible.0.getattr(ENUMS_FIELD)?.downcast_into()?; + let enums: Bound<'py, PyDict> = match source.schema { + Some(schema) => { + let merged: Bound<'py, PyDict> = enums.copy()?; + merged.update(schema_enums(context, schema)?.as_mapping())?; + merged + } + None => enums, + }; + let resolved: EnumColumns<'py> = + enum_columns(context, (&enums, source.visible), source.values, &columns)?; + let fact: ModelSchemaFact = schema_fact(schema_row( + &resolved.columns, + ( + audits.len(), + &context.classes.path_class.call1((source.relative_path,))?, + ), + &families, + (type_enforcement, source.schema.is_some()), + )?); + Ok(AttachedHeader { + description, + schema: Some(HeaderSchema { + columns: resolved.columns, + audits, + dynamic_columns: PyTuple::new(py, families)?, + type_enforcement, + enum_columns: resolved.declarations, + generated_usages: resolved.usages, + model_schema: source.schema.map(|schema| schema.name.clone()), + fact, + }), + }) +} + +/// The header's dynamic column families, each `name_pattern` compiled natively. +fn dynamic_families( + py: Python<'_>, + context: &LoopContext<'_, '_>, + raw: &PyNode<'_>, + model: (&str, &str), +) -> PyResult> { + parse_dynamic_columns(raw, model).map_err(|error| raised_config_error(py, context, error)) +} + +/// The header description: absent, or a non-blank string by Python's `str.strip`. +fn description( + context: &LoopContext<'_, '_>, + source: &HeaderSource<'_, '_>, +) -> PyResult> { + let Some(value) = present(source.values, DESCRIPTION_KEY)? else { + return Ok(None); + }; + if let Ok(text) = value.downcast::() + && text.call_method0(STRIP_METHOD)?.is_truthy()? + { + return Ok(Some(text.to_str()?.to_owned())); + } + let message: String = format!( + "{} model '{DESCRIPTION_KEY}' must be a non-empty string", + source.relative_path + ); + Err(compile_error(context.classes, &message, None)) +} + +fn raised_config_error( + py: Python<'_>, + context: &LoopContext<'_, '_>, + error: sqlbuild_model_config::errors::ConfigError, +) -> pyo3::PyErr { + match native_config_error(py, error) { + Ok(native) => config_error(context.classes, native.bind(py).as_any(), None), + Err(failure) => failure, + } +} + +fn column_migration(column: &Bound<'_, PyAny>) -> PyResult { + Ok(ColumnMigration { + name: column.getattr(NAME_FIELD)?.str()?.to_string(), + migrate_from: column.getattr(MIGRATE_FROM_FIELD)?.extract()?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs new file mode 100644 index 0000000000..f2fbcfe732 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs @@ -0,0 +1,591 @@ +//! A model's hooks, expanded and checked natively. + +use std::cell::RefCell; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, + PyTupleMethods, +}; +use sqlbuild_analysis::compiled_project::main::located_scan_error::located_scan_error; +use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge_call_site_references; +use sqlbuild_analysis::compiled_project::models::ScanFaultOrigin; +use sqlbuild_attachments::audits::main::render_parameterized_sql::render_parameterized_sql; +use sqlbuild_attachments::audits::models::ParameterRenderStop; +use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; + +use crate::bindings::_helpers::attachments::argument_values::argument_pairs; +use crate::bindings::_helpers::compiled_project::model_loop::{ + LoopContext, NativeModelLoop, VisibleDeclarations, compile_error, record_reads, + validation_outcome, +}; +use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_external::external_errors; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ + HOOK_KEYS, present, reference_row, +}; +use crate::bindings::_helpers::compiled_project::model_loop_macros::{ + argument_references, expanded_with, +}; +use crate::bindings::_helpers::compiled_project::model_loop_named::{ + SQL_HOOK_DECLARATION, named_usages, +}; +use crate::bindings::_helpers::compiled_project::model_loop_visibility::ModelVisibility; +use crate::bindings::_helpers::render::declaration_expansion::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, + expand_declarations, +}; +use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; +use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolated}; + +const REFERENCES_METHOD: &str = "references"; +const STATEMENT_ATTRIBUTE: &str = "statement"; +const NAME_ATTRIBUTE: &str = "name"; +const KWARGS_ATTRIBUTE: &str = "kwargs"; +const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; +const DEFINITION_SQL_ATTRIBUTE: &str = "definition_sql"; +const DESCRIPTION_ATTRIBUTE: &str = "description"; +const FILE_PATH_ATTRIBUTE: &str = "file_path"; +const SQL_BODY_ATTRIBUTE: &str = "sql_body"; +const READS_ATTRIBUTE: &str = "reads"; +const KIND_ATTRIBUTE: &str = "kind"; +const FILE_PATH_ARGUMENT: &str = "file_path"; +const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; +const MACRO_TOKEN: char = '@'; +/// `_VisibleModelDeclarations` fields an unscoped hook's `DeclarationResolutionContext` reads. +const UNSCOPED_FIELDS: [&str; 4] = [ + "enums", + "constants", + "inaccessible_enums", + "inaccessible_constants", +]; + +/// What a model's SQL hooks are expanded with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct SqlHookClasses<'py> { + named_sql_hook_entry: Bound<'py, PyAny>, + python_hook_entry: Bound<'py, PyAny>, + /// Discovered SQL hook files by name. + definitions: Bound<'py, PyDict>, + pub(crate) effective_vars: Bound<'py, PyDict>, + /// `UnicodeEnvironment()` and `render_project_var_text`. + pub(crate) environment: Bound<'py, PyAny>, + pub(crate) render_project_var: Bound<'py, PyAny>, + sql_resource_ref: Bound<'py, PyAny>, + /// `SqlResourceRefKind` by the `SqlReferenceKind` value of an explicit reference. + explicit_kinds: Bound<'py, PyDict>, + declaration_identity: Bound<'py, PyAny>, + /// `DeclarationKind.SQL_HOOK`. + sql_hook_kind: Bound<'py, PyAny>, +} + +/// The model whose hooks are expanded. +pub(crate) struct HookModel<'a, 'py> { + pub(crate) name: &'a str, + pub(crate) posix_path: &'a str, + pub(crate) relative_path: &'a str, + pub(crate) file_path: &'a str, + pub(crate) contents: &'a str, + pub(crate) authored_query: &'a str, + pub(crate) visible: &'a VisibleDeclarations<'py>, + pub(crate) visibility: &'a ModelVisibility<'py>, +} + +/// The hook-expanded config values, SQL hook count, and the usages hook SQL recorded. +pub(crate) struct ExpandedHooks<'py> { + pub(crate) values: Bound<'py, PyDict>, + pub(crate) statements: usize, + pub(crate) usages: Vec>, +} + +/// `expand_model_hook_macros_result`, then the hooks' reference and intrinsic checks. +pub(crate) fn expanded_hooks<'py>( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, 'py>, + values: &Bound<'py, PyDict>, + model: &HookModel<'_, 'py>, +) -> PyResult> { + let py: Python<'py> = values.py(); + let context_values: Bound<'py, PyDict> = context + .classes + .config + .builder + .get() + .model_context_values(values, model.name, true)?; + let run: HookRun<'_, '_, 'py> = HookRun { + model_loop, + context, + model, + context_values, + usages: RefCell::new(Vec::new()), + references: RefCell::new(Vec::new()), + }; + let expanded: Bound<'py, PyDict> = values.copy()?; + let model_scope: Scope<'_, 'py> = Scope { + file_path: model.file_path, + consumer: model.visible.consumer.clone(), + }; + for key in HOOK_KEYS { + let Some(hooks) = present(values, key)? else { + continue; + }; + expanded.set_item(key, run.value(&hooks, &model_scope, (Some(key), None))?)?; + } + let mut references: Vec = Vec::new(); + for row in run.references.take() { + if !references.contains(&row) { + references.push(row); + } + } + let external: Vec> = external_errors(context, &references, model.relative_path)?; + let rows: Vec<(&str, &str, bool)> = references + .iter() + .zip(&external) + .map(|((kind, name, _, _), error)| (kind.as_str(), name.as_str(), error.is_some())) + .collect(); + let outcome: Bound<'py, PyAny> = model_loop + .validator + .bind(py) + .call_method1(REFERENCES_METHOD, ((model.name, model.relative_path), rows))?; + validation_outcome(context, &outcome, external, &PyDict::new(py))?; + let statements: usize = reject_hook_intrinsics(model_loop, context, model.name, &expanded)?; + Ok(ExpandedHooks { + values: expanded, + statements, + usages: run.usages.take(), + }) +} + +/// The file a hook's SQL is authored in and the identity its usages are recorded for. +struct Scope<'a, 'py> { + file_path: &'a str, + consumer: Bound<'py, PyAny>, +} + +/// `(hook key, index)` of the entry being expanded, for error labels. +type HookPosition<'a> = (Option<&'a str>, Option); + +/// One model's hook expansion and the facts it collects. +struct HookRun<'a, 'b, 'py> { + model_loop: &'a NativeModelLoop, + context: &'a LoopContext<'b, 'py>, + model: &'a HookModel<'a, 'py>, + context_values: Bound<'py, PyDict>, + usages: RefCell>>, + references: RefCell>, +} + +impl<'py> HookRun<'_, '_, 'py> { + fn classes(&self) -> &SqlHookClasses<'py> { + &self.context.classes.sql_hooks + } + + /// `expand_sql_macros_in_value`: hook container shapes expanded recursively. + fn value( + &self, + value: &Bound<'py, PyAny>, + scope: &Scope<'_, 'py>, + position: HookPosition<'_>, + ) -> PyResult> { + let py: Python<'py> = value.py(); + if let Ok(text) = value.downcast::() { + let expanded: String = self.text(text.to_str()?, scope, position)?; + return Ok(PyString::new(py, &expanded).into_any()); + } + if value.is_instance(&self.context.classes.sql_hook_entry)? { + return self.inline_entry(value, scope, position); + } + if value.is_instance(&self.classes().named_sql_hook_entry)? { + return self.named_entry(value, scope, position); + } + if value.is_instance(&self.classes().python_hook_entry)? { + return Ok(value.clone()); + } + let (key, _) = position; + if let Ok(items) = value.downcast::() { + let mut expanded: Vec> = Vec::with_capacity(items.len()); + for (index, item) in items.iter().enumerate() { + expanded.push(self.value(&item, scope, (key, Some(index)))?); + } + return Ok(PyList::new(py, expanded)?.into_any()); + } + if let Ok(items) = value.downcast::() { + let mut expanded: Vec> = Vec::with_capacity(items.len()); + for (index, item) in items.iter().enumerate() { + expanded.push(self.value(&item, scope, (key, Some(index)))?); + } + return Ok(PyTuple::new(py, expanded)?.into_any()); + } + Ok(value.clone()) + } + + fn inline_entry( + &self, + entry: &Bound<'py, PyAny>, + scope: &Scope<'_, 'py>, + position: HookPosition<'_>, + ) -> PyResult> { + let start: usize = self.references.borrow().len(); + let statement: String = self + .value(&entry.getattr(STATEMENT_ATTRIBUTE)?, scope, position)? + .str()? + .to_str()? + .to_owned(); + self.require_payload(&statement, &inline_label(position), scope)?; + let fields: Bound<'py, PyDict> = PyDict::new(entry.py()); + fields.set_item(STATEMENT_ATTRIBUTE, &statement)?; + for field in [ + NAME_ATTRIBUTE, + RELATIVE_PATH_ATTRIBUTE, + DEFINITION_SQL_ATTRIBUTE, + KWARGS_ATTRIBUTE, + DESCRIPTION_ATTRIBUTE, + ] { + fields.set_item(field, entry.getattr(field)?)?; + } + fields.set_item(READS_ATTRIBUTE, self.reads_since(start)?)?; + self.context.classes.sql_hook_entry.call((), Some(&fields)) + } + + fn named_entry( + &self, + entry: &Bound<'py, PyAny>, + scope: &Scope<'_, 'py>, + position: HookPosition<'_>, + ) -> PyResult> { + let py: Python<'py> = entry.py(); + let name: String = entry.getattr(NAME_ATTRIBUTE)?.extract()?; + let label: String = named_label(&name, position); + let Some(definition) = self.classes().definitions.get_item(&name)? else { + let mut known: Vec = self + .classes() + .definitions + .keys() + .into_iter() + .map(|key| key.extract()) + .collect::>()?; + known.sort(); + let known: String = if known.is_empty() { + "none discovered".to_owned() + } else { + known.join(", ") + }; + let message: String = format!( + "{label} in '{}' references an unknown SQL hook. Discovered SQL hooks: {known}. \ + If this is an inline SQL statement, use inline_sql(\"...\").", + scope.file_path + ); + return Err(compile_error(self.context.classes, &message, None)); + }; + self.usages.borrow_mut().extend(named_usages( + self.context, + (&SQL_HOOK_DECLARATION, &name), + &scope.consumer, + (self.model.name, self.model.relative_path), + )?); + let kwargs: Bound<'py, PyDict> = entry.getattr(KWARGS_ATTRIBUTE)?.downcast_into()?; + let body: String = definition.getattr(SQL_BODY_ATTRIBUTE)?.extract()?; + let definition_path: Bound<'py, PyAny> = definition.getattr(RELATIVE_PATH_ATTRIBUTE)?; + let owner: String = format!("{} {label}", scope.file_path); + let definition_label: String = + format!("SQL hook '{name}' in {}", definition_path.str()?.to_str()?); + let rendered: String = render_parameterized_sql(&body, &argument_pairs(&kwargs)?, true) + .map_err(|stop| { + let message: String = match stop { + ParameterRenderStop::UnusedArguments(names) => format!( + "{owner} passes unknown argument(s) to {definition_label}: {}", + names.join(", ") + ), + ParameterRenderStop::MissingArgument(argument) => { + format!("{owner} is missing argument '{argument}' for {definition_label}") + } + ParameterRenderStop::UnsupportedValue(argument) => format!( + "{owner} {definition_label} argument '{argument}' uses an unsupported value" + ), + }; + compile_error(self.context.classes, &message, None) + })?; + let start: usize = self.references.borrow().len(); + let hook_path: String = definition + .getattr(FILE_PATH_ATTRIBUTE)? + .str()? + .to_str()? + .to_owned(); + let hook_scope: Scope<'_, 'py> = Scope { + file_path: &hook_path, + consumer: self + .classes() + .declaration_identity + .call1((&self.classes().sql_hook_kind, &name))?, + }; + let statement: String = self.text(&rendered, &hook_scope, position)?; + self.require_payload(&statement, &label, scope)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(STATEMENT_ATTRIBUTE, &statement)?; + fields.set_item(NAME_ATTRIBUTE, &name)?; + fields.set_item(RELATIVE_PATH_ATTRIBUTE, definition_path)?; + fields.set_item(DEFINITION_SQL_ATTRIBUTE, &body)?; + fields.set_item(KWARGS_ATTRIBUTE, kwargs.copy()?)?; + fields.set_item( + DESCRIPTION_ATTRIBUTE, + definition.getattr(DESCRIPTION_ATTRIBUTE)?, + )?; + fields.set_item(READS_ATTRIBUTE, self.reads_since(start)?)?; + self.context.classes.sql_hook_entry.call((), Some(&fields)) + } + + fn require_payload( + &self, + statement: &str, + label: &str, + scope: &Scope<'_, 'py>, + ) -> PyResult<()> { + if !blank(statement) { + return Ok(()); + } + let message: String = format!( + "{label} in '{}' must render a non-empty SQL payload", + scope.file_path + ); + Err(compile_error(self.context.classes, &message, None)) + } + + /// `expand_authored_sql_result` and the reference scan of one hook SQL text. + fn text( + &self, + sql: &str, + scope: &Scope<'_, 'py>, + position: HookPosition<'_>, + ) -> PyResult { + if uses_template(sql) { + let message: String = format!( + "{} in '{}' uses unsupported ${{...}} template syntax. Use @@CTX:..., \ + @@ENV:..., or @@project_var inside SQL hooks.", + inline_label(position), + scope.file_path + ); + return Err(compile_error(self.context.classes, &message, None)); + } + let py: Python<'py> = scope.consumer.py(); + let hooks: &SqlHookClasses<'py> = self.classes(); + let (interpolated_sql, reads, error) = interpolated( + self.model_loop.python, + InterpolationSources( + hooks.effective_vars.clone(), + hooks.environment.clone(), + Some(self.context_values.clone()), + hooks.render_project_var.clone(), + ), + sql, + scope.file_path, + )?; + let reads: Vec<(String, String)> = reads + .into_iter() + .map(|(kind, name)| (kind.to_owned(), name)) + .collect(); + record_reads(self.context, &reads)?; + if let Some(message) = error { + return Err(compile_error(self.context.classes, &message, None)); + } + let interpolated_sql: String = interpolated_sql.unwrap_or_else(|| sql.to_owned()); + let declarations: Bound<'py, PyAny> = if self.context.classes.macros.scoped { + self.model + .visibility + .resolved(self.context, scope.file_path)? + } else { + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in UNSCOPED_FIELDS { + fields.set_item(field, self.model.visible.declarations.getattr(field)?)?; + } + self.context + .classes + .macros + .declaration_context + .call((), Some(&fields))? + }; + let expanded: ExpandedDeclarations<'py> = + self.declarations(&interpolated_sql, &declarations, scope)?; + let mut sql: String = expanded.spliced.sql; + let mut usages: Vec> = expanded.usages; + let mut argument_rows: Vec = Vec::new(); + if sql.contains(MACRO_TOKEN) { + let macro_declarations: Bound<'py, PyAny> = if self.context.classes.macros.scoped { + declarations + } else { + py.None().into_bound(py) + }; + let output = expanded_with( + self.context, + (¯o_declarations, &scope.consumer), + scope.file_path, + &sql, + )?; + argument_rows = argument_references(self.context, &output)?; + usages.extend(output.usages.into_iter().map(|usage| usage.into_bound(py))); + sql = output.sql; + } + self.usages.borrow_mut().extend(usages); + let scanned = match extract_sql_references(&sql, &self.model_loop.syntax) { + ReferenceExtraction::Extracted(scanned) => scanned, + ReferenceExtraction::Failed(failure) => { + let origin: ScanFaultOrigin<'_> = ScanFaultOrigin { + relative_path: self.model.posix_path, + contents: self.model.contents, + query_sql: self.model.authored_query, + passes: None, + }; + let message: String = + located_scan_error(&failure, &sql, &origin, &self.model_loop.syntax); + return Err(compile_error(self.context.classes, &message, None)); + } + }; + report_invalid_calls( + self.context, + &self.model_loop.syntax, + &scanned.invalid_calls, + ( + Some(self.model.name), + self.model.relative_path, + self.model.file_path, + self.model.contents, + ), + )?; + self.references + .borrow_mut() + .extend(merge_call_site_references( + scanned.references.iter().map(reference_row).collect(), + argument_rows, + )); + Ok(sql) + } + + fn declarations( + &self, + sql: &str, + declarations: &Bound<'py, PyAny>, + scope: &Scope<'_, 'py>, + ) -> PyResult> { + let py: Python<'py> = scope.consumer.py(); + let (references, stop) = declaration_scan(self.model_loop.python, sql); + let file_keyword: Bound<'py, PyDict> = PyDict::new(py); + file_keyword.set_item( + FILE_PATH_ARGUMENT, + self.context.classes.path_class.call1((scope.file_path,))?, + )?; + let render_constant: Bound<'py, PyAny> = self.context.classes.partial.call( + (self.context.host.getattr(CONSTANT_TEXT_CALLBACK)?,), + Some(&file_keyword), + )?; + let classes: ExpansionClasses<'py> = ExpansionClasses { + usage_record: self.context.classes.usage_record.clone(), + compile_error: self.context.classes.compile_error.clone(), + }; + expand_declarations(&DeclarationExpansion { + sql, + references: &references, + stop, + context: declarations, + consumer: ExpansionConsumer { + identity: Some(scope.consumer.clone()), + relationship_visibility: false, + path: scope.file_path, + }, + render_constant: &render_constant, + classes: &classes, + }) + } + + /// `resource_references`: the model, source and seed references read since `start`, once each. + fn reads_since(&self, start: usize) -> PyResult> { + let hooks: &SqlHookClasses<'py> = self.classes(); + let py: Python<'py> = hooks.explicit_kinds.py(); + let mut seen: Vec<(String, String)> = Vec::new(); + let mut reads: Vec> = Vec::new(); + for (kind, name, _, _) in self.references.borrow().iter().skip(start) { + let Some(resource_kind) = hooks.explicit_kinds.get_item(kind)? else { + continue; + }; + if seen.contains(&(kind.clone(), name.clone())) { + continue; + } + seen.push((kind.clone(), name.clone())); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(KIND_ATTRIBUTE, resource_kind)?; + fields.set_item(NAME_ATTRIBUTE, name)?; + reads.push(hooks.sql_resource_ref.call((), Some(&fields))?); + } + PyTuple::new(py, reads) + } +} + +/// Whether Python's `str.strip()` leaves nothing. +fn blank(statement: &str) -> bool { + statement.chars().all(is_python_space) +} + +/// `\$\{[^}]+\}`: a `${` followed by at least one character before its closing `}`. +fn uses_template(sql: &str) -> bool { + sql.match_indices("${").any(|(start, _)| { + let rest: &str = &sql[start + 2..]; + rest.find('}').is_some_and(|end| end > 0) + }) +} + +fn inline_label(position: HookPosition<'_>) -> String { + match position { + (None, _) => "inline_sql(\"...\") hook".to_owned(), + (Some(key), None) => format!("{key} inline_sql(\"...\")"), + (Some(key), Some(index)) => format!("{key}[{index}] inline_sql(\"...\")"), + } +} + +fn named_label(name: &str, position: HookPosition<'_>) -> String { + match position { + (None, _) => format!("sql(\"{name}\") hook"), + (Some(key), None) => format!("{key} sql(\"{name}\")"), + (Some(key), Some(index)) => format!("{key}[{index}] sql(\"{name}\")"), + } +} + +/// Reject cursor intrinsics in each SQL hook statement; return how many statements were checked. +fn reject_hook_intrinsics( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, '_>, + name: &str, + values: &Bound<'_, PyDict>, +) -> PyResult { + let mut statements: usize = 0; + for key in HOOK_KEYS { + let Some(hooks) = present(values, key)? else { + continue; + }; + if !(hooks.is_instance_of::() || hooks.is_instance_of::()) { + continue; + } + let hook_context: String = format!("Model '{name}' {key}"); + for entry in hooks.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if !entry.is_instance(&context.classes.sql_hook_entry)? { + continue; + } + statements += 1; + let statement: String = entry.getattr(STATEMENT_ATTRIBUTE)?.extract()?; + if let IntrinsicCheck::Rejected(message) = intrinsic_free( + model_loop.python, + &statement, + &model_loop.reserved_markers, + &hook_context, + ) { + return Err(compile_error(context.classes, &message, None)); + } + } + } + Ok(statements) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs new file mode 100644 index 0000000000..7b7c331464 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -0,0 +1,285 @@ +//! The Python host the loop hands finished models and user-code steps to. + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use sqlbuild_analysis::compiled_project::main::plain_model_facts::plain_model_facts; +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, ModelConfigFacts, PlainModel, +}; + +use crate::bindings::_helpers::compiled_project::model_input_view::{ + ExpansionRecord, InputProjections, ModelInputRecord, SchemaRecord, +}; +use crate::bindings::_helpers::compiled_project::model_loop_header::AttachedHeader; +use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ + reference_fact, stripped_values, +}; +use crate::bindings::_helpers::compiled_project::project::{ + CONTRACT_KEY, INCREMENTAL_MODE_KEY, materialization, text_value, +}; +use crate::bindings::_helpers::compiled_project::project::{ + DeclarationPy, config_values, declarations, +}; +use crate::bindings::_helpers::render::declaration_expansion::{ + ExpandedDeclarations, ScannedReference, span_rows, +}; +use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; +use crate::bindings::_helpers::render::macro_expansion::ExpansionOutput; + +const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; +const LAYER_SCHEMA_ATTRIBUTE: &str = "layer_schema"; +const VALUES_ARGUMENT: &str = "values"; +const ENUMS_ARGUMENT: &str = "enums"; +const CONSTANTS_ARGUMENT: &str = "constants"; +const LOCAL_ENUMS_ATTRIBUTE: &str = "local_enums"; +const LOCAL_CONSTANTS_ATTRIBUTE: &str = "local_constants"; +const NAME_ATTRIBUTE: &str = "name"; + +/// `((name, POSIX path, relative path, file path), column locations, query, contents, scan)`. +#[derive(FromPyObject)] +pub(crate) struct ModelLoopRow( + pub(crate) (String, String, String, String), + /// Header column locations, header values, and the model's own enums and constants. + pub(crate) (Py, Py, Py, Py), + pub(crate) String, + pub(crate) String, + pub(crate) Option<(Vec, Option)>, + /// The `@@`-interpolated query, its template reads, its error, and its substitution spans. + pub(crate) InterpolationRow, + /// The discovered model file, owned by the discovery inputs. + pub(crate) Py, +); +/// `(interpolated query, template reads, error, substitution spans)`. +pub(crate) type InterpolationRow = (String, Vec<(String, String)>, Option, Vec); +/// `(kind, name, package, call argument count)`. +pub(crate) type ReferenceRow = (String, String, Option, Option); + +/// One model the native loop finished, before the host builds its Python view. +pub(crate) struct FinishedModel<'py> { + pub(crate) config: Bound<'py, PyAny>, + /// The hook-expanded config values, header metadata still included. + pub(crate) values: Bound<'py, PyDict>, + pub(crate) query_sql: String, + pub(crate) header: AttachedHeader<'py>, + pub(crate) references: Vec, + pub(crate) declarations: ExpandedDeclarations<'py>, + pub(crate) hook_statements: usize, + /// Whether the SQL is analysed, and where a rejected `sql_analysis false` was authored. + pub(crate) analysis: (bool, Option>), + /// The usages of the model's named schema and Python hooks, recorded after its hook usages. + pub(crate) named_usages: Vec>, + /// The query's macro expansion, when it calls macros. + pub(crate) macros: Option, +} + +/// The Python classes a finished model's record is built with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct RecordClasses<'py> { + /// `dataclasses.replace`. + replace: Bound<'py, PyAny>, + /// `declaration_rows`, the facts rows of the model's local enums and constants. + declaration_rows: Bound<'py, PyAny>, +} + +/// What the loop knows about a finished model besides its loop row. +pub(crate) struct RecordInputs<'a, 'py> { + pub(crate) classes: &'a RecordClasses<'py>, + /// The visible declarations, for the model's own enums and constants. + pub(crate) visible: &'a Bound<'py, PyAny>, +} + +/// A natively finished model's input record, and the facts the project retains. +pub(crate) fn finished_record<'py>( + inputs: &RecordInputs<'_, 'py>, + row: ModelLoopRow, + finished: FinishedModel<'py>, +) -> PyResult<(ModelInputRecord, CompiledModelFacts)> { + let py: Python<'py> = inputs.visible.py(); + let ModelLoopRow( + (name, relative_path, _, _), + _, + authored_query_sql, + authored_sql, + _, + (interpolated_sql, _, _, interpolation_spans), + model_file, + ) = row; + let stripped: Bound<'py, PyDict> = stripped_values(py, &finished.values)?; + let mut usages: Vec> = finished.declarations.usages; + if let Some(schema) = &finished.header.schema { + usages.extend(schema.generated_usages.iter().cloned()); + } + let enum_columns: Vec = match &finished.header.schema { + Some(schema) => schema.enum_columns.keys().extract()?, + None => Vec::new(), + }; + let local_enums: Bound<'py, PyTuple> = + dict_values(&inputs.visible.getattr(LOCAL_ENUMS_ATTRIBUTE)?)?; + let local_constants: Bound<'py, PyTuple> = + dict_values(&inputs.visible.getattr(LOCAL_CONSTANTS_ATTRIBUTE)?)?; + let rows_arguments: Bound<'py, PyDict> = PyDict::new(py); + rows_arguments.set_item(ENUMS_ARGUMENT, &local_enums)?; + rows_arguments.set_item(CONSTANTS_ARGUMENT, &local_constants)?; + let (enums, constants): (Vec>, Vec>) = inputs + .classes + .declaration_rows + .call((), Some(&rows_arguments))? + .extract()?; + let replaced: Bound<'py, PyDict> = PyDict::new(py); + replaced.set_item(VALUES_ARGUMENT, &stripped)?; + let config: Bound<'py, PyAny> = inputs + .classes + .replace + .call((&finished.config,), Some(&replaced))?; + let mut declaration_usages: Vec> = unique(py, usages)?; + declaration_usages.extend(finished.named_usages); + let (macro_sql, macro_spans, macro_deps, macro_usages) = match finished.macros { + Some(macros) => { + let mut names: Vec = Vec::new(); + for dependency in unique( + py, + macros + .dependencies + .into_iter() + .map(|item| item.into_bound(py)) + .collect(), + )? { + names.push(dependency.getattr(NAME_ATTRIBUTE)?.extract()?); + } + ( + macros.sql, + macros + .spans + .into_iter() + .map(span_row) + .collect::>(), + names, + unique( + py, + macros + .usages + .into_iter() + .map(|item| item.into_bound(py)) + .collect(), + )?, + ) + } + None => ( + finished.declarations.spliced.sql.clone(), + Vec::new(), + Vec::new(), + Vec::new(), + ), + }; + let schema: Option = finished.header.schema.as_ref().map(|schema| SchemaRecord { + description: finished.header.description.clone(), + columns: schema.columns.clone().unbind(), + audits: schema.audits.clone().unbind(), + dynamic_columns: schema.dynamic_columns.clone().unbind(), + type_enforcement: schema.type_enforcement, + model_schema: schema.model_schema.clone(), + }); + let discovered_query: Py = model_file + .bind(py) + .getattr("query_sql")? + .downcast_into::()? + .unbind(); + let authored: Py = shared_sql(py, &authored_query_sql, &[&discovered_query])?; + let macro_source: Py = + shared_sql(py, &finished.declarations.spliced.sql, &[&authored])?; + let expanded: Py = shared_sql(py, ¯o_sql, &[&authored, ¯o_source])?; + let query: Py = shared_sql( + py, + &finished.query_sql, + &[&authored, ¯o_source, &expanded], + )?; + let record: ModelInputRecord = ModelInputRecord { + model_file: model_file.clone_ref(py), + config: config.unbind(), + query_sql: query, + macro_source_sql: macro_source, + references: finished.references.clone(), + schema, + sql_validation_enabled: finished.analysis.0, + rejected_opt_out: finished.analysis.1.map(Bound::unbind), + enum_declarations: local_enums.unbind(), + constant_declarations: local_constants.unbind(), + enum_columns: finished + .header + .schema + .as_ref() + .map_or_else(|| PyDict::new(py), |schema| schema.enum_columns.clone()) + .unbind(), + macro_deps, + macro_usages: PyTuple::new(py, macro_usages)?.unbind(), + declaration_usages: PyTuple::new(py, unique(py, declaration_usages)?)?.unbind(), + expansion: ExpansionRecord { + authored: interpolated_sql == authored_query_sql, + authored_sql: authored, + expanded_sql: expanded, + interpolation_spans, + declaration_spans: span_rows(&finished.declarations.spliced), + macro_spans, + }, + projections: InputProjections::default(), + compiled: None, + }; + let (values, encoding_error) = config_values(&stripped); + let facts: CompiledModelFacts = plain_model_facts(PlainModel { + name, + relative_path, + query_sql: finished.query_sql, + authored_query_sql, + authored_sql, + config: ModelConfigFacts { + values, + encoding_error, + header_keys: finished.config.getattr(HEADER_KEYS_ATTRIBUTE)?.extract()?, + layer_schema: finished.config.getattr(LAYER_SCHEMA_ATTRIBUTE)?.extract()?, + contract: text_value(&stripped, CONTRACT_KEY)?, + materialized: materialization(&stripped)?, + incremental_mode: text_value(&stripped, INCREMENTAL_MODE_KEY)?, + }, + schema: finished.header.schema.map(|schema| schema.fact), + references: finished + .references + .into_iter() + .map(reference_fact) + .collect(), + enum_columns, + enum_declarations: declarations(enums)?, + constant_declarations: declarations(constants)?, + }); + Ok((record, facts)) +} + +/// Reuse an identical immutable spelling already owned by this model. +fn shared_sql(py: Python<'_>, sql: &str, candidates: &[&Py]) -> PyResult> { + for candidate in candidates { + if candidate.bind(py).to_str()? == sql { + return Ok(candidate.clone_ref(py)); + } + } + Ok(PyString::new(py, sql).unbind()) +} + +/// A dict's values in insertion order. +fn dict_values<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { + let mapping: &Bound<'py, PyDict> = mapping.downcast()?; + PyTuple::new(py_of(mapping), mapping.values()) +} + +fn py_of<'py>(mapping: &Bound<'py, PyDict>) -> Python<'py> { + mapping.py() +} + +/// `dict.fromkeys(items)`: each item once, first occurrence first, by Python equality. +fn unique<'py>(py: Python<'py>, items: Vec>) -> PyResult>> { + let seen: Bound<'py, PyDict> = PyDict::new(py); + for item in items { + seen.set_item(item, py.None())?; + } + Ok(seen.keys().into_iter().collect()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs new file mode 100644 index 0000000000..0b935b2035 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs @@ -0,0 +1,83 @@ +//! Reading a model's config values the way the attachment reads them. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyDictMethods}; +use sqlbuild_analysis::compiled_project::main::header_metadata_keys::is_header_metadata_key; +use sqlbuild_analysis::compiled_project::models::{AnalysisOverride, ModelReferenceFact}; +use sqlbuild_sqltext::sql_references::models::SqlReference; + +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; + +const SQL_ANALYSIS_KEY: &str = "sql_analysis"; +const SQL_VALIDATION_KEY: &str = "sql_validation"; +pub(crate) const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; + +/// Whether either hook key holds a value. +pub(crate) fn hook_presence(values: &Bound<'_, PyDict>) -> PyResult { + for key in HOOK_KEYS { + if present(values, key)?.is_some() { + return Ok(true); + } + } + Ok(false) +} + +/// The override `_is_sql_validation_enabled` reads: unequal keys conflict, a non-bool inherits. +pub(crate) fn analysis_override(values: &Bound<'_, PyDict>) -> PyResult { + let analysis: Option> = present(values, SQL_ANALYSIS_KEY)?; + let legacy: Option> = present(values, SQL_VALIDATION_KEY)?; + let chosen: Option> = match (analysis, legacy) { + (Some(analysis), Some(legacy)) => { + if analysis.ne(&legacy)? { + return Ok(AnalysisOverride::Conflict); + } + Some(analysis) + } + (Some(value), None) | (None, Some(value)) => Some(value), + (None, None) => None, + }; + Ok(match chosen.map(|value| value.downcast_into::()) { + Some(Ok(flag)) if flag.is_true() => AnalysisOverride::Enabled, + Some(Ok(_)) => AnalysisOverride::Disabled, + Some(Err(_)) | None => AnalysisOverride::Inherit, + }) +} + +/// The value under `key`, treating an explicit `None` as absent. +pub(crate) fn present<'py>( + values: &Bound<'py, PyDict>, + key: &str, +) -> PyResult>> { + Ok(values.get_item(key)?.filter(|value| !value.is_none())) +} + +/// The config without header metadata, always a new dictionary as hook expansion leaves it. +pub(crate) fn stripped_values<'py>( + py: Python<'py>, + values: &Bound<'py, PyDict>, +) -> PyResult> { + let stripped: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in values.iter() { + if !is_header_metadata_key(&key.str()?.to_string()) { + stripped.set_item(key, value)?; + } + } + Ok(stripped) +} + +pub(crate) fn reference_row(reference: &SqlReference) -> ReferenceRow { + ( + reference.kind.to_owned(), + reference.name.clone(), + reference.package.clone(), + reference.call_argument_count, + ) +} + +pub(crate) fn reference_fact((kind, name, package, _): ReferenceRow) -> ModelReferenceFact { + ModelReferenceFact { + kind, + name, + package, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs new file mode 100644 index 0000000000..fbf83c25e3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs @@ -0,0 +1,221 @@ +//! A model query's macro calls, expanded natively with the model's visible declarations. + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_analysis::compiled_project::main::authored_model_offset::authored_model_offset; +use sqlbuild_analysis::compiled_project::main::text_position::text_position; +use sqlbuild_analysis::compiled_project::models::{SplicedSpan, TextPosition}; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::compiled_project::model_loop::{ + LoopContext, VisibleDeclarations, compile_error, +}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::model_loop_inputs::reference_row; +use crate::bindings::_helpers::render::macro_expansion::{ + ExpansionOutput, NativeMacroExpansion, expanded, +}; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; + +/// Fields of `_VisibleModelDeclarations` a `DeclarationResolutionContext` is built from. +const CONTEXT_FIELDS: [&str; 9] = [ + "enums", + "constants", + "inaccessible_enums", + "inaccessible_constants", + "enum_visibility", + "constant_visibility", + "macros", + "macro_records", + "inaccessible_macros", +]; +const ENFORCE_ATTRIBUTE: &str = "_enforce_explicit_references"; + +/// What every model's macro expansion reads: the project's macros, context, bridge and classes. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct MacroClasses<'py> { + loaded_macros: Py, + macro_context: Bound<'py, PyAny>, + /// The active `MacroBridge`, or `None`. + bridge: Bound<'py, PyAny>, + /// Whether a declaration scope resolver indexes the project, so declarations are scoped. + pub(crate) scoped: bool, + pub(crate) declaration_context: Bound<'py, PyAny>, + macro_argument_error: Bound<'py, PyAny>, + expansion: Py, + pub(crate) generic_syntax: LexicalSyntaxInput, +} + +/// The model whose query is expanded: its paths, authored text and declaration passes. +pub(crate) struct MacroModel<'a> { + pub(crate) file_path: &'a str, + pub(crate) relative_path: &'a str, + pub(crate) contents: &'a str, + pub(crate) authored_query: &'a str, + /// Whether interpolation left the authored query unchanged, so offsets map to the file. + pub(crate) authored: bool, + pub(crate) declaration_spans: &'a [SplicedSpan], +} + +/// The expanded query, its call-site references, and the expansion facts the host keeps. +pub(crate) struct ExpandedMacros { + pub(crate) references: Vec, + pub(crate) output: ExpansionOutput, +} + +/// `_macro_expansion`: expand the query, locating argument errors in the authored file. +pub(crate) fn expanded_macros<'py>( + context: &LoopContext<'_, 'py>, + visible: &VisibleDeclarations<'py>, + sql: &str, + model: &MacroModel<'_>, +) -> PyResult { + let classes: &MacroClasses<'py> = &context.classes.macros; + let py: Python<'py> = visible.consumer.py(); + let declarations: Bound<'py, PyAny> = if classes.scoped { + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in CONTEXT_FIELDS { + fields.set_item(field, visible.declarations.getattr(field)?)?; + } + fields.set_item("consumer", &visible.consumer)?; + classes.declaration_context.call((), Some(&fields))? + } else { + py.None().into_bound(py) + }; + let output: ExpansionOutput = match expanded_with( + context, + (&declarations, &visible.consumer), + model.file_path, + sql, + ) { + Ok(output) => output, + Err(error) => return Err(located(context, error, (sql, model))), + }; + let references: Vec = argument_references(context, &output)?; + Ok(ExpandedMacros { references, output }) +} + +/// `expand_sql_macros_result`: expand `sql` with the declarations (or `None`) and consumer. +pub(crate) fn expanded_with<'py>( + context: &LoopContext<'_, 'py>, + reader: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + file_path: &str, + sql: &str, +) -> PyResult { + let (declarations, consumer) = reader; + let classes: &MacroClasses<'py> = &context.classes.macros; + let py: Python<'py> = consumer.py(); + let expansion: NativeMacroExpansion = NativeMacroExpansion::new( + &context.classes.path_class.call1((file_path,))?, + ( + classes.loaded_macros.clone_ref(py), + PyDict::new(py).unbind(), + classes.macro_context.clone().unbind(), + declarations.clone().unbind(), + consumer.clone().unbind(), + ), + classes + .macro_context + .getattr(ENFORCE_ATTRIBUTE)? + .is_truthy()?, + classes.expansion.clone_ref(py), + )?; + let expansion: Bound<'py, NativeMacroExpansion> = Bound::new(py, expansion)?; + expanded(&expansion, sql, &classes.bridge) +} + +/// `call_site_sql_references`: typed macro arguments as the generic SQL reference scan reads them. +pub(crate) fn argument_references( + context: &LoopContext<'_, '_>, + output: &ExpansionOutput, +) -> PyResult> { + if output.argument_references.is_empty() { + return Ok(Vec::new()); + } + let functions: &Bound<'_, PyDict> = &context.classes.macros.ref_functions()?; + let mut calls: Vec = Vec::with_capacity(output.argument_references.len()); + for (kind, name) in &output.argument_references { + let function: String = functions + .get_item(kind)? + .map_or_else(|| Ok(String::new()), |found| found.extract())?; + calls.push(format!("{function}(\"{name}\")")); + } + let syntax: LexicalSyntax = context.classes.macros.generic_syntax.clone().into(); + match extract_sql_references(&calls.join(" "), &syntax) { + ReferenceExtraction::Extracted(scanned) => { + Ok(scanned.references.iter().map(reference_row).collect()) + } + ReferenceExtraction::Failed(failure) => { + Err(compile_error(context.classes, &failure.message, None)) + } + } +} + +impl<'py> MacroClasses<'py> { + /// The names of the project's loaded macros. + pub(crate) fn macro_names(&self) -> PyResult> { + let py: Python<'py> = self.macro_context.py(); + self.loaded_macros + .bind(py) + .iter() + .map(|(name, _)| name.extract()) + .collect() + } + + /// Typed reference function by `SqlResourceRefKind` value, from the expansion classes. + fn ref_functions(&self) -> PyResult> { + let py: Python<'py> = self.macro_context.py(); + Ok(self + .expansion + .bind(py) + .get_item("ref_functions")? + .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err("ref_functions"))? + .downcast_into::()?) + } +} + +/// A `MacroArgumentError` at its authored file location, when its offset is authored text. +fn located( + context: &LoopContext<'_, '_>, + error: PyErr, + expansion: (&str, &MacroModel<'_>), +) -> PyErr { + let (sql, model) = expansion; + let classes: &MacroClasses<'_> = &context.classes.macros; + let py: Python<'_> = classes.macro_context.py(); + if !model.authored || !error.is_instance(py, &classes.macro_argument_error) { + return error; + } + let relocated = (|| -> PyResult> { + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let offset: usize = value.getattr("offset")?.extract()?; + let passes: Vec> = vec![model.declaration_spans.to_vec()]; + let Some(authored) = + authored_model_offset(model.contents, model.authored_query, (sql, offset), &passes) + else { + return Ok(None); + }; + let position: TextPosition = text_position(model.contents, authored); + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item( + "path", + context.classes.path_class.call1((model.relative_path,))?, + )?; + fields.set_item("line", position.line)?; + fields.set_item("column", position.column)?; + let location: Bound<'_, PyAny> = + context.contracts.source_location.call((), Some(&fields))?; + let moved: Bound<'_, PyAny> = value.call_method1("located", (location,))?; + moved.setattr("__suppress_context__", true)?; + Ok(Some(PyErr::from_value(moved))) + })(); + match relocated { + Ok(Some(moved)) => moved, + Ok(None) => error, + Err(failure) => failure, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_named.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_named.rs new file mode 100644 index 0000000000..6a237a47f2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_named.rs @@ -0,0 +1,232 @@ +//! `named_declaration_usages`: a named declaration's usage, reporting S006 when it is hidden. + +use std::path::Path; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::path_visible::path_visible; +use sqlbuild_scopes::scope_index::models::{ + DeclaredScope, ResourceIdentity, ResourceKind, ScopeKind, ScopePathFailure, +}; + +use crate::bindings::_helpers::compiled_project::model_loop::LoopContext; + +const NAME_FIELD: &str = "name"; +const PATH_FIELD: &str = "path"; +const SCOPE_FIELD: &str = "scope"; +const VALUE_FIELD: &str = "value"; +const IDENTITY_FIELD: &str = "identity"; +const OWNER_FIELD: &str = "owner"; +const KIND_FIELD: &str = "kind"; +const OWNING_PATH_FIELD: &str = "owning_path"; +const OWNERSHIP_ROOT_FIELD: &str = "ownership_root"; +const CONSUMER_FIELD: &str = "consumer"; +const DECLARATION_FIELD: &str = "declaration"; +const CURRENT_FOLDER: &str = "."; +const INACCESSIBLE_CODE: &str = "S006"; +const MODEL_RESOURCE: &str = "model"; + +/// A named declaration kind: its enum value, diagnostic label and declaration role. +pub(crate) struct NamedKind { + pub(crate) value: &'static str, + pub(crate) label: &'static str, + pub(crate) role: &'static str, +} + +/// Reusable model schemas. +pub(crate) const SCHEMA_DECLARATION: NamedKind = NamedKind { + value: "schema", + label: "Schema", + role: "schemas", +}; +/// Named SQL lifecycle hooks. +pub(crate) const SQL_HOOK_DECLARATION: NamedKind = NamedKind { + value: "sql_hook", + label: "SQL hook", + role: "hooks/sql", +}; +/// Python lifecycle hooks. +pub(crate) const PYTHON_HOOK_DECLARATION: NamedKind = NamedKind { + value: "python_hook", + label: "Python hook", + role: "hooks/python", +}; + +/// Generic audits attached to models, sources and seeds. +pub(crate) const AUDIT_DECLARATION: NamedKind = NamedKind { + value: "audit", + label: "Generic audit", + role: "audits/generic", +}; + +/// The usage of the indexed declaration `name`, if any, reporting S006 when it is hidden. +pub(crate) fn named_usages<'py>( + context: &LoopContext<'_, 'py>, + declaration: (&NamedKind, &str), + consumer: &Bound<'py, PyAny>, + model: (&str, &str), +) -> PyResult>> { + let (model_name, relative_path) = model; + resource_named_usages( + context, + declaration, + consumer, + (MODEL_RESOURCE, model_name, relative_path), + ) +} + +/// `named_usages` for a consumer of `(resource type, name, relative path)`. +pub(crate) fn resource_named_usages<'py>( + context: &LoopContext<'_, 'py>, + declaration: (&NamedKind, &str), + consumer: &Bound<'py, PyAny>, + resource: (&str, &str, &str), +) -> PyResult>> { + let (kind, name) = declaration; + let (resource_type, model_name, relative_path) = resource; + let py: Python<'py> = consumer.py(); + let key: Bound<'py, PyTuple> = PyTuple::new(py, [kind.value, name])?; + let Some(record) = context.classes.named_records.get_item(key)? else { + return Ok(Vec::new()); + }; + let label: String = relative_path.replace(std::path::MAIN_SEPARATOR, "/"); + let declared: DeclaredScope = declared_scope(&record)?; + let Ok(visible) = path_visible(&declared, &label) else { + return Err(path_error(context, &declared, &label)); + }; + if !visible { + report_inaccessible( + context, + &record, + (kind, name), + (resource_type, model_name, &label), + )?; + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(CONSUMER_FIELD, consumer)?; + fields.set_item(DECLARATION_FIELD, record.getattr(IDENTITY_FIELD)?)?; + Ok(vec![context.classes.usage_record.call((), Some(&fields))?]) +} + +/// `InvalidScopePathError` for the consumer path or owner folder Python normalizes first. +fn path_error(context: &LoopContext<'_, '_>, declared: &DeclaredScope, label: &str) -> PyErr { + match context + .classes + .scope_path_error + .call1((scope_path_message(declared, label),)) + { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// The `InvalidScopePathError` message of the first path Python's visibility check rejects. +pub(crate) fn scope_path_message(declared: &DeclaredScope, label: &str) -> String { + let owner: &str = declared + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(CURRENT_FOLDER); + let (path, failure) = match normalize_scope_path(label) { + Err(failure) => (label, failure), + Ok(_) => match normalize_scope_path(owner) { + Err(failure) => (owner, failure), + Ok(_) => (label, ScopePathFailure::NotProjectRelative), + }, + }; + match failure { + ScopePathFailure::NotProjectRelative => { + format!("Scope path must be project-relative: {path}") + } + ScopePathFailure::EscapesProject => format!("Scope path escapes the project: {path}"), + } +} + +/// The scope facts of a Python `DeclarationRecord`. +fn declared_scope(record: &Bound<'_, PyAny>) -> PyResult { + let scope: String = record + .getattr(SCOPE_FIELD)? + .getattr(VALUE_FIELD)? + .extract()?; + let owner: Bound<'_, PyAny> = record.getattr(IDENTITY_FIELD)?.getattr(OWNER_FIELD)?; + let owner: Option = if owner.is_none() { + None + } else { + let kind: String = owner.getattr(KIND_FIELD)?.getattr(VALUE_FIELD)?.extract()?; + let name: String = owner.getattr(NAME_FIELD)?.extract()?; + ResourceKind::parse(&kind).map(|kind| ResourceIdentity { kind, name }) + }; + Ok(DeclaredScope { + scope: ScopeKind::parse(&scope).unwrap_or(ScopeKind::Global), + owner, + owning_path: record.getattr(OWNING_PATH_FIELD)?.extract()?, + }) +} + +/// Report Python's S006 diagnostic for a declaration the model's folder cannot see. +fn report_inaccessible( + context: &LoopContext<'_, '_>, + record: &Bound<'_, PyAny>, + declaration: (&NamedKind, &str), + resource: (&str, &str, &str), +) -> PyResult<()> { + let py: Python<'_> = record.py(); + let (kind, name) = declaration; + let (resource_type, model_name, label) = resource; + let folder: &str = Path::new(label) + .parent() + .and_then(Path::to_str) + .filter(|parent| !parent.is_empty()) + .unwrap_or(CURRENT_FOLDER); + let owning_path: Bound<'_, PyAny> = record.getattr(OWNING_PATH_FIELD)?; + let owner: String = if owning_path.is_truthy()? { + owning_path.str()?.to_string() + } else { + record + .getattr(OWNERSHIP_ROOT_FIELD)? + .getattr(PATH_FIELD)? + .str()? + .to_string() + }; + let record_path: String = record.getattr(PATH_FIELD)?.str()?.to_string(); + let scope: String = record + .getattr(SCOPE_FIELD)? + .getattr(VALUE_FIELD)? + .extract()?; + let (label_text, role) = (kind.label, kind.role); + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", "error")?; + fields.set_item("code", INACCESSIBLE_CODE)?; + fields.set_item( + "message", + format!( + "{label_text} '{name}' is not visible from '{label}'. It is defined at {record_path} \ + with {scope} scope owned by '{owner}'" + ), + )?; + fields.set_item("resource_type", resource_type)?; + fields.set_item("resource_name", model_name)?; + fields.set_item("path", context.classes.path_class.call1((label,))?)?; + fields.set_item( + "help", + format!( + "move '{record_path}' to a {role}/ role owned by '{folder}' or one of its parent \ + folders, or to the project-wide {role}/ when consumers span resource trees" + ), + )?; + let diagnostic: Bound<'_, PyAny> = context + .classes + .compiler_diagnostic + .call((), Some(&fields))?; + let key: Bound<'_, PyTuple> = PyTuple::new(py, [INACCESSIBLE_CODE, label, kind.value, name])?; + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("key", key)?; + arguments.set_item("diagnostic", diagnostic)?; + context + .classes + .report_diagnostic + .call((), Some(&arguments))?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs new file mode 100644 index 0000000000..3e9fa876e4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs @@ -0,0 +1,230 @@ +//! `rejected_sql_analysis_opt_out`: `sql_analysis false` on SQL that parses, when it is required. + +use std::path::Path; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyBoolMethods, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; +use sqlbuild_analysis::compiled_project::main::opt_out_position::opt_out_position; +use sqlbuild_analysis::compiled_project::models::TextPosition; +use sqlbuild_attachments::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; + +use crate::bindings::_helpers::compiled_project::model_loop::{ + LoopContext, NativeModelLoop, compile_error, +}; +use crate::bindings::_helpers::compiled_project::model_loop_inputs::{HOOK_KEYS, present}; +use crate::bindings::_helpers::project_assembly::resources::{syntax_error_message, syntax_valid}; + +const OPT_OUT_KEYS: [&str; 2] = ["sql_analysis", "sql_validation"]; +const CURSOR_TYPE_KEY: &str = "cursor_type"; +const PLACEHOLDERS_KEY: &str = "placeholders"; +const INTEGER_CURSOR: &str = "integer"; +const INTEGER_LITERAL: &str = "CAST(0 AS BIGINT)"; +const TIMESTAMP_LITERAL: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; +const INTRINSIC_CONTEXT: &str = "Model SQL"; +const GENERIC_DIALECT: &str = "generic"; +const STATEMENT_ATTRIBUTE: &str = "statement"; +const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; +const PATH_DEFAULT_ATTRIBUTE: &str = "matched_path_default"; +const PATHLIB_MODULE: &str = "pathlib"; +const PATH_CLASS: &str = "Path"; +const PATH_FIELD: &str = "path"; +const LINE_FIELD: &str = "line"; +const COLUMN_FIELD: &str = "column"; +const HELP_ARGUMENT: &str = "help"; + +/// Where the opt-out was authored, or a check the attachment still answers. +pub(crate) enum OptOut<'py> { + Accepted, + Rejected(Bound<'py, PyAny>), +} + +/// The model the opt-out is checked for. +pub(crate) struct OptOutModel<'a, 'py> { + pub(crate) name: &'a str, + pub(crate) file_path: &'a str, + pub(crate) config: &'a Bound<'py, PyAny>, + pub(crate) values: &'a Bound<'py, PyDict>, + pub(crate) query_sql: &'a str, + pub(crate) relative_path: &'a str, + pub(crate) contents: &'a str, +} + +/// Whether the opt-out is rejected, and where it was authored. +pub(crate) fn rejected_opt_out<'py>( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, 'py>, + model: &OptOutModel<'_, 'py>, +) -> PyResult> { + let settings = model_loop.settings; + if !settings.require_sql_analysis + || !settings.sql_analysis + || settings.no_sql_validation + || !opted_out(model.values)? + { + return Ok(OptOut::Accepted); + } + let mut statements: Vec = vec![analysis_sql(model_loop, context, model)?]; + for key in HOOK_KEYS { + if let Some(hooks) = model.values.get_item(key)? { + statements.extend(hook_statements(context, &hooks)?); + } + } + let placeholders: Vec<(String, String)> = placeholders(model.values)?; + let checks: Vec = statements + .into_iter() + .map(|sql| SyntaxCheck { + sql, + placeholders: placeholders.clone(), + }) + .collect(); + if syntax_valid(model.config.py(), GENERIC_DIALECT, &checks)? { + Ok(OptOut::Rejected(location(model_loop, context, model)?)) + } else { + Ok(OptOut::Accepted) + } +} + +/// `validate_sql_syntax`: raise the model's SQL syntax error before its references are scanned. +pub(crate) fn reject_unparseable_query( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, '_>, + model: &OptOutModel<'_, '_>, +) -> PyResult<()> { + let check: SyntaxCheck = SyntaxCheck { + sql: analysis_sql(model_loop, context, model)?, + placeholders: placeholders(model.values)?, + }; + let Some(error) = syntax_error_message( + model.config.py(), + &check, + GENERIC_DIALECT, + SyntaxMode::Validate, + )? + else { + return Ok(()); + }; + let message: String = format!( + "SQL syntax error in model '{}' ({}): {error}", + model.name, model.file_path + ); + let keywords: Bound<'_, PyDict> = PyDict::new(model.config.py()); + keywords.set_item(HELP_ARGUMENT, &context.classes.syntax_help)?; + Err(PyErr::from_value( + context + .classes + .compile_error + .call((message,), Some(&keywords))?, + )) +} + +/// `cursor_intrinsics_analysis_sql`: the query with each cursor intrinsic as a typed literal. +fn analysis_sql( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, '_>, + model: &OptOutModel<'_, '_>, +) -> PyResult { + let literal: &str = match model.values.get_item(CURSOR_TYPE_KEY)? { + Some(value) if value.eq(INTEGER_CURSOR)? => INTEGER_LITERAL, + _ => TIMESTAMP_LITERAL, + }; + let (analysis_sql, _) = replace_intrinsics( + model_loop.python, + model.query_sql, + INTRINSIC_CONTEXT, + |_| literal.to_owned(), + ) + .map_err(|message| compile_error(context.classes, &message, None))?; + Ok(analysis_sql) +} + +/// Whether either opt-out key is exactly `False`. +fn opted_out(values: &Bound<'_, PyDict>) -> PyResult { + for key in OPT_OUT_KEYS { + if let Some(value) = values.get_item(key)? + && value.downcast::().is_ok_and(|flag| !flag.is_true()) + { + return Ok(true); + } + } + Ok(false) +} + +/// `hook_sql_statements`: strings, SQL hook statements, and nested lists or tuples, in order. +fn hook_statements( + context: &LoopContext<'_, '_>, + value: &Bound<'_, PyAny>, +) -> PyResult> { + if let Ok(text) = value.downcast::() { + return Ok(vec![text.extract()?]); + } + if value.is_instance(&context.classes.sql_hook_entry)? { + return hook_statements(context, &value.getattr(STATEMENT_ATTRIBUTE)?); + } + let mut statements: Vec = Vec::new(); + if value.is_instance_of::() || value.is_instance_of::() { + for item in value.try_iter()? { + statements.extend(hook_statements(context, &item?)?); + } + } + Ok(statements) +} + +/// The model's `placeholders` mapping as `str` pairs, when it is a dictionary. +fn placeholders(values: &Bound<'_, PyDict>) -> PyResult> { + let Some(raw) = present(values, PLACEHOLDERS_KEY)? else { + return Ok(Vec::new()); + }; + let Ok(raw) = raw.downcast::() else { + return Ok(Vec::new()); + }; + raw.iter() + .map(|(key, value)| Ok((key.str()?.to_string(), value.str()?.to_string()))) + .collect() +} + +/// `_opt_out_location`: the model header when it sets the key, else its path default entry. +fn location<'py>( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, 'py>, + model: &OptOutModel<'_, 'py>, +) -> PyResult> { + let py: Python<'py> = model.config.py(); + let header_keys: Vec = model.config.getattr(HEADER_KEYS_ATTRIBUTE)?.extract()?; + let (path, position): (&str, TextPosition) = if header_keys + .iter() + .any(|key| OPT_OUT_KEYS.contains(&key.as_str())) + { + ( + model.relative_path, + opt_out_position(model_loop.python, model.contents, 0), + ) + } else { + let path_key: Option = model.config.getattr(PATH_DEFAULT_ATTRIBUTE)?.extract()?; + let config_path: &Path = Path::new(&model_loop.project_config_path); + let config_text: String = if config_path.is_file() { + std::fs::read_to_string(config_path)? + } else { + String::new() + }; + let start: usize = path_key + .filter(|key| !key.is_empty()) + .and_then(|key| config_text.find(&key)) + .map_or(0, |byte| config_text[..byte].chars().count()); + let file_name: &str = model_loop + .project_config_path + .rsplit('/') + .next() + .unwrap_or_default(); + ( + file_name, + opt_out_position(model_loop.python, &config_text, start), + ) + }; + let path_class: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr(PATH_CLASS)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(PATH_FIELD, path_class.call1((path,))?)?; + fields.set_item(LINE_FIELD, position.line)?; + fields.set_item(COLUMN_FIELD, position.column)?; + context.contracts.source_location.call((), Some(&fields)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_python_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_python_hooks.rs new file mode 100644 index 0000000000..a3ccac90e9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_python_hooks.rs @@ -0,0 +1,258 @@ +//! A model's Python lifecycle hooks: checked against their functions' signatures, and their usages. + +use std::cell::RefCell; +use std::collections::HashMap; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; + +use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compile_error}; +use crate::bindings::_helpers::compiled_project::model_loop_named::{ + PYTHON_HOOK_DECLARATION, named_usages, +}; + +/// Hook keys in the sorted order `validate_python_hook_config` reads them. +const SORTED_HOOK_KEYS: [&str; 2] = ["post_hooks", "pre_hooks"]; +/// Parameters SQLBuild injects with the hook context. +const CONTEXT_PARAMETERS: [&str; 4] = ["ctx", "context", "_ctx", "hook_context"]; +const NAME_FIELD: &str = "name"; +const KWARGS_FIELD: &str = "kwargs"; +const FUNCTION_FIELD: &str = "function"; +const POSITIONAL_ONLY: u8 = 0; +const POSITIONAL_OR_KEYWORD: u8 = 1; +const KEYWORD_ONLY: u8 = 3; +const VAR_KEYWORD: u8 = 4; + +/// One parameter as `inspect.signature` reports it: name, `Parameter.kind`, and a default. +#[derive(Clone, FromPyObject)] +struct HookParameter(String, u8, bool); + +/// The discovered hook functions, provider names, `PythonHookEntry`, and signature reader. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct HookClasses<'py> { + /// Discovered hook functions by name. + functions: Bound<'py, PyDict>, + provider_names: Vec, + python_hook_entry: Bound<'py, PyAny>, + /// Reads a hook function's parameters with `inspect.signature`. + signature: Bound<'py, PyAny>, +} + +/// Each hook function's parameters, read once per compile. +#[derive(Default)] +pub(crate) struct HookSignatures { + by_name: RefCell>>, +} + +/// One Python hook entry where it was authored. +struct HookSite<'a> { + model_name: &'a str, + hook_key: &'a str, + index: usize, + name: &'a str, +} + +impl HookSignatures { + /// `_checked_python_hooks`: validate each Python hook, then return their usages in order. + pub(crate) fn checked<'py>( + &self, + context: &LoopContext<'_, 'py>, + values: &Bound<'py, PyDict>, + consumer: &Bound<'py, PyAny>, + model: (&str, &str), + ) -> PyResult>> { + let (model_name, _) = model; + let entries: Vec<(&str, usize, Bound<'py, PyAny>)> = python_entries(context, values)?; + let functions: &Bound<'py, PyDict> = &context.classes.hooks.functions; + for (hook_key, index, entry) in &entries { + let name: String = entry.getattr(NAME_FIELD)?.extract()?; + let site: HookSite<'_> = HookSite { + model_name, + hook_key, + index: *index, + name: &name, + }; + let Some(function) = functions.get_item(&name)? else { + let mut known: Vec = functions.keys().extract()?; + known.sort_unstable(); + let known: String = if known.is_empty() { + "none discovered".to_owned() + } else { + known.join(", ") + }; + let message: String = format!( + "{} references an unknown hook. Discovered hooks: {known}", + site.prefix() + ); + return Err(compile_error(context.classes, &message, None)); + }; + let kwargs: Vec = entry + .getattr(KWARGS_FIELD)? + .downcast_into::()? + .keys() + .extract()?; + let parameters: Vec = self.parameters(context, &name, &function)?; + if let Some(message) = + signature_problem(&site, ¶meters, &kwargs, &context.classes.hooks) + { + return Err(compile_error(context.classes, &message, None)); + } + } + let mut usages: Vec> = Vec::new(); + for (_, _, entry) in &entries { + let name: String = entry.getattr(NAME_FIELD)?.extract()?; + usages.extend(named_usages( + context, + (&PYTHON_HOOK_DECLARATION, &name), + consumer, + model, + )?); + } + Ok(usages) + } + + /// The hook function's parameters, read through `inspect.signature` the first time. + fn parameters( + &self, + context: &LoopContext<'_, '_>, + name: &str, + function: &Bound<'_, PyAny>, + ) -> PyResult> { + if let Some(parameters) = self.by_name.borrow().get(name) { + return Ok(parameters.clone()); + } + let parameters: Vec = context + .classes + .hooks + .signature + .call1((function.getattr(FUNCTION_FIELD)?,))? + .extract()?; + self.by_name + .borrow_mut() + .insert(name.to_owned(), parameters.clone()); + Ok(parameters) + } +} + +impl HookSite<'_> { + fn prefix(&self) -> String { + format!( + "model '{}' {}[{}] python(\"{}\")", + self.model_name, self.hook_key, self.index, self.name + ) + } +} + +/// Every `PythonHookEntry` of the sorted hook keys, with its key and index. +fn python_entries<'py>( + context: &LoopContext<'_, 'py>, + values: &Bound<'py, PyDict>, +) -> PyResult)>> { + let mut entries: Vec<(&'static str, usize, Bound<'py, PyAny>)> = Vec::new(); + for hook_key in SORTED_HOOK_KEYS { + let Some(raw) = values.get_item(hook_key)? else { + continue; + }; + if !(raw.is_instance_of::() || raw.is_instance_of::()) { + continue; + } + for (index, entry) in raw.try_iter()?.enumerate() { + let entry: Bound<'py, PyAny> = entry?; + if entry.is_instance(&context.classes.hooks.python_hook_entry)? { + entries.push((hook_key, index, entry)); + } + } + } + Ok(entries) +} + +/// `validate_python_hook_signature`: the first problem with the configured arguments, if any. +fn signature_problem( + site: &HookSite<'_>, + parameters: &[HookParameter], + kwargs: &[String], + classes: &HookClasses<'_>, +) -> Option { + let prefix: String = site.prefix(); + let context_parameter = |name: &str| CONTEXT_PARAMETERS.contains(&name); + let provider = |name: &str| classes.provider_names.iter().any(|known| known == name); + let mut sorted_kwargs: Vec<&str> = kwargs.iter().map(String::as_str).collect(); + sorted_kwargs.sort_unstable(); + if let Some(conflict) = sorted_kwargs.iter().find(|name| context_parameter(name)) { + return Some(format!( + "{prefix} argument '{conflict}' conflicts with reserved context parameter \ + '{conflict}'. Rename the hook argument; context parameters are injected by SQLBuild." + )); + } + if let Some(conflict) = sorted_kwargs.iter().find(|name| provider(name)) { + return Some(format!( + "{prefix} argument '{conflict}' conflicts with provider injection for parameter \ + '{conflict}'. Rename the hook argument or remove it to let SQLBuild inject the \ + provider." + )); + } + let keyword = |parameter: &HookParameter| { + (parameter.1 == POSITIONAL_OR_KEYWORD || parameter.1 == KEYWORD_ONLY) + && !context_parameter(¶meter.0) + }; + let accepts_var_keyword: bool = parameters + .iter() + .any(|parameter| parameter.1 == VAR_KEYWORD); + let mut accepted: Vec<&str> = parameters + .iter() + .filter(|parameter| keyword(parameter)) + .map(|parameter| parameter.0.as_str()) + .collect(); + accepted.sort_unstable(); + accepted.dedup(); + let unknown: Vec<&str> = sorted_kwargs + .iter() + .copied() + .filter(|name| !accepts_var_keyword && !accepted.contains(name)) + .collect(); + if !unknown.is_empty() { + let accepted: String = if accepted.is_empty() { + "none".to_owned() + } else { + accepted.join(", ") + }; + return Some(format!( + "{prefix} has unknown argument(s): {}. Accepted configured arguments: {accepted}", + unknown.join(", ") + )); + } + let required_positional_only: Vec<&str> = parameters + .iter() + .filter(|parameter| { + parameter.1 == POSITIONAL_ONLY && !parameter.2 && !context_parameter(¶meter.0) + }) + .map(|parameter| parameter.0.as_str()) + .collect(); + if !required_positional_only.is_empty() { + return Some(format!( + "{prefix} cannot be configured because the hook function has required \ + positional-only parameter(s): {}. Use keyword-capable parameters for values \ + supplied from MODEL hooks.", + required_positional_only.join(", ") + )); + } + let missing: Vec<&str> = parameters + .iter() + .filter(|parameter| { + keyword(parameter) + && !parameter.2 + && !provider(¶meter.0) + && !kwargs.contains(¶meter.0) + }) + .map(|parameter| parameter.0.as_str()) + .collect(); + if !missing.is_empty() { + return Some(format!( + "{prefix} is missing required argument(s): {}", + missing.join(", ") + )); + } + None +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs new file mode 100644 index 0000000000..39cdd7bfef --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs @@ -0,0 +1,219 @@ +//! A model's `model_schema`: resolved, checked for visibility, and merged into its header columns. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; + +use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compile_error}; +use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; + +const MODEL_SCHEMA_KEY: &str = "model_schema"; +const NAME_FIELD: &str = "name"; +const TYPE_FIELD: &str = "type"; +const NULLABLE_FIELD: &str = "nullable"; +const DESCRIPTION_FIELD: &str = "description"; +const AUDITS_FIELD: &str = "audits"; +const MIGRATE_FROM_FIELD: &str = "migrate_from"; +const LOCATION_FIELD: &str = "location"; +const PATH_FIELD: &str = "path"; +const LINE_FIELD: &str = "line"; +const COLUMNS_FIELD: &str = "columns"; +const LOWER_METHOD: &str = "lower"; +const STRIP_METHOD: &str = "strip"; +/// `SchemaColumn` fields a named-schema column keeps when a header column augments it. +const KEPT_NAMED_FIELDS: [&str; 6] = [ + "name", + "type", + "nullable", + "description", + "meta", + "location", +]; + +/// The reusable schema a model names: its columns, name, description and enum column types. +pub(crate) struct ModelSchema<'py> { + pub(crate) declaration: Bound<'py, PyAny>, + pub(crate) name: String, +} + +/// `_resolve_model_schema`: `None`, or the public schema the model's config names. +pub(crate) fn model_schema<'py>( + context: &LoopContext<'_, 'py>, + values: &Bound<'py, PyDict>, + model_name: &str, +) -> PyResult>> { + let Some(raw) = present(values, MODEL_SCHEMA_KEY)? else { + return Ok(None); + }; + let Ok(name) = raw.downcast::() else { + return Err(invalid_name(context, model_name)); + }; + if !name.call_method0(STRIP_METHOD)?.is_truthy()? { + return Err(invalid_name(context, model_name)); + } + let schemas: &Bound<'py, PyDict> = &context.classes.model_schemas; + let Some(declaration) = schemas.get_item(name)? else { + let mut available: Vec = schemas.keys().extract()?; + available.sort_unstable(); + let listed: String = if available.is_empty() { + "none".to_owned() + } else { + available.join(", ") + }; + let message: String = format!( + "model '{model_name}' references unknown model_schema '{name}'; available schemas: \ + {listed}" + ); + return Err(compile_error(context.classes, &message, None)); + }; + Ok(Some(ModelSchema { + declaration, + name: name.extract()?, + })) +} + +/// The schema's column names, which validation reads as the declared columns. +pub(crate) fn schema_column_names(schema: &ModelSchema<'_>) -> PyResult> { + schema + .declaration + .getattr(COLUMNS_FIELD)? + .try_iter()? + .map(|column| column?.getattr(NAME_FIELD)?.extract()) + .collect() +} + +/// `model_schema_enum_declarations`: the schema's column types that name a project enum. +pub(crate) fn schema_enums<'py>( + context: &LoopContext<'_, 'py>, + schema: &ModelSchema<'py>, +) -> PyResult> { + let py: Python<'py> = schema.declaration.py(); + let enums: Bound<'py, PyDict> = PyDict::new(py); + for column in schema.declaration.getattr(COLUMNS_FIELD)?.try_iter()? { + let column_type: Bound<'py, PyAny> = column?.getattr(TYPE_FIELD)?; + if column_type.is_none() { + continue; + } + if let Some(declaration) = context.classes.schema_enums.get_item(&column_type)? { + enums.set_item(column_type, declaration)?; + } + } + Ok(enums) +} + +/// `_merge_model_schema_columns`: named columns first, augmented by header audits, then the rest. +pub(crate) fn merged_columns<'py>( + context: &LoopContext<'_, 'py>, + schema: &ModelSchema<'py>, + local: &Bound<'py, PyTuple>, + model: (&str, &str), +) -> PyResult> { + let py: Python<'py> = local.py(); + let mut merged: Vec> = schema + .declaration + .getattr(COLUMNS_FIELD)? + .try_iter()? + .collect::>()?; + let mut keys: Vec> = Vec::with_capacity(merged.len()); + for column in &merged { + keys.push(column.getattr(NAME_FIELD)?.call_method0(LOWER_METHOD)?); + } + let mut additional: Vec> = Vec::new(); + for column in local.iter() { + let key: Bound<'py, PyAny> = column.getattr(NAME_FIELD)?.call_method0(LOWER_METHOD)?; + let mut named_index: Option = None; + for (index, candidate) in keys.iter().enumerate() { + if candidate.eq(&key)? { + named_index = Some(index); + } + } + let Some(index) = named_index else { + additional.push(column); + continue; + }; + check_augmentation(context, &column, &merged[index], model)?; + merged[index] = augmented(context, &merged[index], &column)?; + } + merged.extend(additional); + PyTuple::new(py, merged) +} + +fn augmented<'py>( + context: &LoopContext<'_, 'py>, + named: &Bound<'py, PyAny>, + local: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = named.py(); + let mut audits: Vec> = named + .getattr(AUDITS_FIELD)? + .try_iter()? + .collect::>()?; + for audit in local.getattr(AUDITS_FIELD)?.try_iter()? { + let audit: Bound<'py, PyAny> = audit?; + let mut present: bool = false; + for existing in &audits { + if existing.eq(&audit)? { + present = true; + break; + } + } + if !present { + audits.push(audit); + } + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in KEPT_NAMED_FIELDS { + fields.set_item(field, named.getattr(field)?)?; + } + fields.set_item(AUDITS_FIELD, PyTuple::new(py, audits)?)?; + fields.set_item(MIGRATE_FROM_FIELD, local.getattr(MIGRATE_FROM_FIELD)?)?; + context.contracts.schema_column.call((), Some(&fields)) +} + +/// `_validate_model_schema_audit_augmentation`. +fn check_augmentation( + context: &LoopContext<'_, '_>, + local: &Bound<'_, PyAny>, + named: &Bound<'_, PyAny>, + model: (&str, &str), +) -> PyResult<()> { + let (model_name, relative_path) = model; + let name: String = local.getattr(NAME_FIELD)?.str()?.to_string(); + let location: Bound<'_, PyAny> = named.getattr(LOCATION_FIELD)?; + let origin: String = if location.is_none() { + "the named schema".to_owned() + } else { + format!( + "{}:{}", + location.getattr(PATH_FIELD)?.str()?, + location.getattr(LINE_FIELD)?.str()? + ) + }; + let mut overridden: Vec<&str> = Vec::new(); + for field in [TYPE_FIELD, NULLABLE_FIELD, DESCRIPTION_FIELD] { + if !local.getattr(field)?.is_none() { + overridden.push(field); + } + } + if !overridden.is_empty() { + let message: String = format!( + "model '{model_name}' in {relative_path} cannot override {} for named-schema column \ + '{name}' from {origin}; only audit augmentation is supported", + overridden.join(", ") + ); + return Err(compile_error(context.classes, &message, None)); + } + if !local.getattr(AUDITS_FIELD)?.is_truthy()? && local.getattr(MIGRATE_FROM_FIELD)?.is_none() { + let message: String = format!( + "model '{model_name}' in {relative_path} redeclares named-schema column '{name}' from \ + {origin} without audits or migrate_from; only audit augmentation and column \ + migrations are supported" + ); + return Err(compile_error(context.classes, &message, None)); + } + Ok(()) +} + +fn invalid_name(context: &LoopContext<'_, '_>, model_name: &str) -> pyo3::PyErr { + let message: String = format!("model '{model_name}': model_schema must be a non-empty string"); + compile_error(context.classes, &message, None) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs new file mode 100644 index 0000000000..448d8ab7b3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs @@ -0,0 +1,281 @@ +//! The declarations a model sees: `_VisibleModelDeclarationCache.for_model`, natively. + +use std::cell::{Cell, RefCell}; +use std::collections::HashMap; +use std::path::Path; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; + +use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, VisibleDeclarations}; +use crate::bindings::_helpers::scopes::declaration_contexts::NativeDeclarationContexts; + +const NAME_FIELD: &str = "name"; +const ENUMS_FIELD: &str = "enums"; +const CONSTANTS_FIELD: &str = "constants"; +const ENUM_VISIBILITY_FIELD: &str = "enum_visibility"; +const CONSTANT_VISIBILITY_FIELD: &str = "constant_visibility"; +/// `_VisibleModelDeclarations` fields taken unchanged from the resolved context. +const SHARED_FIELDS: [&str; 5] = [ + "inaccessible_enums", + "inaccessible_constants", + "macros", + "macro_records", + "inaccessible_macros", +]; +/// Every `_VisibleModelDeclarations` field, in declaration order. +const VISIBLE_FIELDS: [&str; 11] = [ + "local_enums", + "local_constants", + ENUMS_FIELD, + CONSTANTS_FIELD, + "inaccessible_enums", + "inaccessible_constants", + ENUM_VISIBILITY_FIELD, + CONSTANT_VISIBILITY_FIELD, + "macros", + "macro_records", + "inaccessible_macros", +]; + +/// The scope resolver, project declarations and Python classes model visibility reads. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct VisibilityClasses<'py> { + /// The `DeclarationScopeResolver`, or `None`. + pub(crate) resolver: Bound<'py, PyAny>, + /// Whether the resolver indexes declarations, so contexts are projected from it. + pub(crate) projects: bool, + /// Models whose private declarations or grants make their context their own. + resource_specific: Bound<'py, PyAny>, + pub(crate) public_enums: Bound<'py, PyDict>, + pub(crate) public_constants: Bound<'py, PyDict>, + visible_declarations: Bound<'py, PyAny>, + pub(crate) resource_identity: Bound<'py, PyAny>, + declaration_identity: Bound<'py, PyAny>, + model_kind: Bound<'py, PyAny>, + enum_kind: Bound<'py, PyAny>, + constant_kind: Bound<'py, PyAny>, + private_owner: Bound<'py, PyAny>, + visibility_record: Bound<'py, PyAny>, +} + +/// The model whose declarations are resolved: name, file path, and its own enums and constants. +pub(crate) struct VisibilityModel<'a, 'py> { + pub(crate) name: &'a str, + pub(crate) file_path: &'a str, + pub(crate) local: (&'a Bound<'py, PyAny>, &'a Bound<'py, PyAny>), +} + +/// The per-folder declarations of one compile and the contexts projected for them. +#[derive(Default)] +pub(crate) struct ModelVisibility<'py> { + by_parent: RefCell>>, + projected: Cell, +} + +impl<'py> ModelVisibility<'py> { + /// `for_model`: the model's visible declarations and the identity its usages record. + pub(crate) fn for_model( + &self, + context: &LoopContext<'_, 'py>, + model: &VisibilityModel<'_, 'py>, + ) -> PyResult> { + let classes: &VisibilityClasses<'py> = &context.classes.visibility; + let consumer: Bound<'py, PyAny> = classes + .resource_identity + .call1((&classes.model_kind, model.name))?; + let parent: String = Path::new(model.file_path) + .parent() + .map(|parent| parent.to_string_lossy().into_owned()) + .unwrap_or_default(); + let reusable: bool = !classes.resolver.is_none(); + let declarations: Bound<'py, PyAny> = if classes.resource_specific.contains(&consumer)? { + self.built(context, model, &consumer)? + } else if let Some(cached) = self.by_parent.borrow().get(&parent).filter(|_| reusable) { + rebound(context, cached, &consumer)? + } else { + let built: Bound<'py, PyAny> = self.built(context, model, &consumer)?; + if reusable { + self.by_parent.borrow_mut().insert(parent, built.clone()); + } + built + }; + Ok(VisibleDeclarations { + declarations, + consumer, + }) + } + + /// The number of declaration contexts the loop projected rather than reused. + pub(crate) fn projected(&self) -> usize { + self.projected.get() + } + + /// `_build_visible_declaration_indexes`. + fn built( + &self, + context: &LoopContext<'_, 'py>, + model: &VisibilityModel<'_, 'py>, + consumer: &Bound<'py, PyAny>, + ) -> PyResult> { + let classes: &VisibilityClasses<'py> = &context.classes.visibility; + let py: Python<'py> = consumer.py(); + let local_enums: Bound<'py, PyDict> = by_name(model.local.0)?; + let local_constants: Bound<'py, PyDict> = by_name(model.local.1)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("local_enums", &local_enums)?; + fields.set_item("local_constants", &local_constants)?; + let (enums, constants) = if classes.projects { + let resolved: Bound<'py, PyAny> = self.resolved(context, model.file_path)?; + for field in SHARED_FIELDS { + fields.set_item(field, resolved.getattr(field)?)?; + } + fields.set_item( + ENUM_VISIBILITY_FIELD, + resolved.getattr(ENUM_VISIBILITY_FIELD)?, + )?; + fields.set_item( + CONSTANT_VISIBILITY_FIELD, + resolved.getattr(CONSTANT_VISIBILITY_FIELD)?, + )?; + ( + resolved.getattr(ENUMS_FIELD)?.downcast_into::()?, + resolved + .getattr(CONSTANTS_FIELD)? + .downcast_into::()?, + ) + } else { + for field in [ + SHARED_FIELDS[0], + SHARED_FIELDS[1], + ENUM_VISIBILITY_FIELD, + CONSTANT_VISIBILITY_FIELD, + SHARED_FIELDS[2], + SHARED_FIELDS[3], + SHARED_FIELDS[4], + ] { + fields.set_item(field, PyDict::new(py))?; + } + ( + classes.public_enums.clone(), + classes.public_constants.clone(), + ) + }; + fields.set_item(ENUMS_FIELD, merged(&enums, &local_enums)?)?; + fields.set_item(CONSTANTS_FIELD, merged(&constants, &local_constants)?)?; + for (field, kind, local) in [ + (ENUM_VISIBILITY_FIELD, &classes.enum_kind, &local_enums), + ( + CONSTANT_VISIBILITY_FIELD, + &classes.constant_kind, + &local_constants, + ), + ] { + let inherited: Bound<'py, PyDict> = fields.get_item(field)?.map_or_else( + || Ok(PyDict::new(py)), + |value| value.downcast_into::(), + )?; + let visibility: Bound<'py, PyDict> = inherited.copy()?; + for name in local.keys() { + let record: Bound<'py, PyAny> = private_record(context, (kind, &name), consumer)?; + visibility.set_item(name, PyTuple::new(py, [record])?)?; + } + fields.set_item(field, visibility)?; + } + classes.visible_declarations.call((), Some(&fields)) + } + + /// `resolve_declaration_context` for an authored file. + pub(crate) fn resolved( + &self, + context: &LoopContext<'_, 'py>, + file_path: &str, + ) -> PyResult> { + let resolver: &Bound<'py, PyAny> = &context.classes.visibility.resolver; + let contexts: Bound<'py, NativeDeclarationContexts> = + resolver.getattr("native_contexts")?.downcast_into()?; + let (resolved, projected) = contexts.get().resolved_context(resolver, file_path, None)?; + if projected { + self.projected.set(self.projected.get() + 1); + } + Ok(resolved) + } +} + +/// `_rebind_visible_declarations`: the folder's declarations with visibility rebound. +fn rebound<'py>( + context: &LoopContext<'_, 'py>, + cached: &Bound<'py, PyAny>, + consumer: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = cached.py(); + let enum_visibility: Bound<'py, PyDict> = + cached.getattr(ENUM_VISIBILITY_FIELD)?.downcast_into()?; + let constant_visibility: Bound<'py, PyDict> = + cached.getattr(CONSTANT_VISIBILITY_FIELD)?.downcast_into()?; + if enum_visibility.is_empty() && constant_visibility.is_empty() { + return Ok(cached.clone()); + } + let resolver: &Bound<'py, PyAny> = &context.classes.visibility.resolver; + let contexts: Bound<'py, NativeDeclarationContexts> = + resolver.getattr("native_contexts")?.downcast_into()?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in VISIBLE_FIELDS { + fields.set_item(field, cached.getattr(field)?)?; + } + fields.set_item( + ENUM_VISIBILITY_FIELD, + contexts + .get() + .rebound_visibility(&enum_visibility, consumer)?, + )?; + fields.set_item( + CONSTANT_VISIBILITY_FIELD, + contexts + .get() + .rebound_visibility(&constant_visibility, consumer)?, + )?; + context + .classes + .visibility + .visible_declarations + .call((), Some(&fields)) +} + +/// A model-private declaration's `VisibilityRecord`, visible to its owner. +fn private_record<'py>( + context: &LoopContext<'_, 'py>, + declaration: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + consumer: &Bound<'py, PyAny>, +) -> PyResult> { + let classes: &VisibilityClasses<'py> = &context.classes.visibility; + let (kind, name) = declaration; + let identity: Bound<'py, PyAny> = classes.declaration_identity.call1((kind, name, consumer))?; + let fields: Bound<'py, PyDict> = PyDict::new(consumer.py()); + fields.set_item("reason", &classes.private_owner)?; + classes + .visibility_record + .call((consumer, identity), Some(&fields)) +} + +/// Declarations by name, later ones replacing earlier ones. +fn by_name<'py>(declarations: &Bound<'py, PyAny>) -> PyResult> { + let named: Bound<'py, PyDict> = PyDict::new(declarations.py()); + for declaration in declarations.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + named.set_item(declaration.getattr(NAME_FIELD)?, declaration)?; + } + Ok(named) +} + +/// `base | overlay`. +fn merged<'py>( + base: &Bound<'py, PyDict>, + overlay: &Bound<'py, PyDict>, +) -> PyResult> { + let merged: Bound<'py, PyDict> = base.copy()?; + merged.update(overlay.as_mapping())?; + Ok(merged) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs new file mode 100644 index 0000000000..bd519a1933 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -0,0 +1,775 @@ +//! The compile's natively owned project, which producers record into and consumers read. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Mutex; + +use crate::bindings::_helpers::compiled_project::resource_diagnostics::{ + DiagnosticSettings, ResourceDiagnostics, resource_diagnostics, +}; +use crate::bindings::_helpers::compiled_project::scenario_loop::ScenarioRow; +use crate::bindings::_helpers::compiled_project::sql_test_loop::{ + AttachedSqlTests, SqlTestSettings, TestFileRow, +}; + +use pyo3::exceptions::{PyIndexError, PyRuntimeError}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{FromPyObject, pyclass, pymethods}; +use serde_json::Value; +use sqlbuild_analysis::compiled_project::main::project_models::project_models; +use sqlbuild_analysis::compiled_project::main::record_model::record_model; +use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, CompiledProjectFacts, DeclarationFact, DynamicFamilyFact, DynamicProofFact, + ModelAnalysisFact, ModelConfigFacts, ModelReferenceFact, ModelSchemaFact, SchemaColumnFact, +}; + +use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_guard}; +use crate::bindings::_helpers::compiled_project::audit_loop::{ + AttachmentOwner, AuditFileRow, AuditSettings, model_owner, resource_owner, +}; +use crate::bindings::_helpers::compiled_project::audit_views::{ + AuditRecord, NativeAuditInput, NativeCompiledAudit, +}; +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + CompiledViewClasses, InferredRow, NativeCompiledModel, +}; +use crate::bindings::_helpers::compiled_project::function_loop::{FunctionRow, FunctionSettings}; +use crate::bindings::_helpers::compiled_project::function_views::{ + AttachedFunction, FunctionRecord, NativeCompiledFunction, NativeFunctionInput, ObjectKeyRow, +}; +use crate::bindings::_helpers::compiled_project::inputs_view::NativeInputsView; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + ModelInputRecord, NativeModelInput, +}; +use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; +use crate::bindings::_helpers::compiled_project::seed_views::{ + NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedRecord, paired_seeds, +}; +use crate::bindings::_helpers::compiled_project::source_loop::{SourceRow, SourceSettings}; +use crate::bindings::_helpers::compiled_project::source_views::{ + AttachedSource, NativeCompiledSource, NativeSourceInput, SourceNamespace, SourceRecord, +}; +use crate::bindings::_helpers::compiled_project::values::{object_value, plain_value}; + +pub(crate) const CONTRACT_KEY: &str = "contract"; +const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; +const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; +const SOURCE_OWNER: &str = "source"; +const SEED_OWNER: &str = "seed"; +const ATTACHMENT_LOOP_UNSET: &str = "no model loop was kept to attach this project's resources"; +pub(crate) const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; +const MATERIALIZED_KEY: &str = "materialized"; +const TABLE_MATERIALIZATION: &str = "table"; + +/// `(name, type)`. +type NamedType = (String, Option); +/// `(name, type, nullable, audit count, declared outside the model)`. +pub(crate) type SchemaColumnPy = (String, Option, Option, u32, bool); +/// `(key, name, pivot column, value column, aggregate, type, name pattern)`. +pub(crate) type DynamicFamilyPy = ( + String, + String, + String, + String, + String, + String, + Option, +); +/// `(entry audit count, columns, dynamic families, (type enforcement, named schema))`. +pub(crate) type SchemaPy = (u32, Vec, Vec, (bool, bool)); +/// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. +type ProofPy = (bool, bool, Vec, Option, Vec); +/// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. +type AnalysisPy = (bool, Vec, Vec, bool, Vec); + +/// Encoded config values and the encoder's error, if any. +pub(crate) type ConfigValues = (BTreeMap, Option); + +/// One model as the compile-input stage produced it. +#[derive(FromPyObject)] +pub(crate) struct ModelInputPy<'py>( + (String, String, String, String, String), + (Bound<'py, PyDict>, Vec, Option), + Vec<(String, String, Option)>, + Option, + Vec, + (Vec>, Vec>), +); + +/// Name, relative path, members, value, value type and rendering of one declaration. +#[derive(FromPyObject)] +pub(crate) struct DeclarationPy<'py>( + String, + String, + Vec<(String, Bound<'py, PyAny>)>, + Bound<'py, PyAny>, + Option, + Option, +); + +/// One compile's project facts, recorded by the stages that produce them. +#[pyclass(module = "sqlbuild._native")] +#[derive(Default)] +pub(crate) struct NativeCompiledProject { + facts: CompiledProjectFacts, + /// What the model loop finished for each model beyond its facts, in loop order. + inputs: Vec, + /// The Python classes model input views project their attributes with. + input_classes: Option>, + /// Shared immutable column projections, allocated only when Python consumers request them. + inferred_column_projections: Mutex>>, + /// Lineage graphs its project views built. + graph_builds: usize, + /// Each attached seed, in attachment order. + seeds: Vec, + /// Each attached source, in attachment order. + sources: Vec, + /// Each attached SQL, then Python, function in attachment order. + functions: Vec, + /// Each singular, then model-, source- and seed-attached, audit in attachment order. + audits: Vec, + /// The model loop and its host, which attach the project's other resources. + attachment: Option<(Py, Py)>, +} + +impl NativeCompiledProject { + /// The facts recorded so far, for native consumers such as the rules request. + pub(crate) fn facts(&self) -> &CompiledProjectFacts { + &self.facts + } + + /// Retain facts a producer finished natively. + pub(crate) fn retain(&mut self, facts: CompiledModelFacts) { + record_model(&mut self.facts, facts); + } + + /// Retain a finished model's input record; return its index for the model's view. + pub(crate) fn retain_input(&mut self, record: ModelInputRecord) -> usize { + self.inputs.push(record); + self.inputs.len() - 1 + } + + /// Keep the classes model input views project their attributes with. + pub(crate) fn set_input_classes(&mut self, classes: Py) { + self.input_classes = Some(classes); + } + + /// Count one lineage graph a project view built. + pub(crate) fn record_graph_build(&mut self) { + self.graph_builds += 1; + } + + /// Each schema-attached model, then each source and seed, with the audits attached to it. + fn attachment_owners<'py>(&self, py: Python<'py>) -> PyResult>> { + let mut owners: Vec> = Vec::new(); + for record in &self.inputs { + let Some(schema) = &record.schema else { + continue; + }; + let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); + owners.push(model_owner( + model_file + .getattr("file_path")? + .getattr("stem")? + .extract()?, + model_file + .getattr(RELATIVE_PATH_ATTRIBUTE)? + .str()? + .to_string(), + (schema.audits.bind(py), schema.columns.bind(py)), + )?); + } + for source in &self.sources { + let attached: &AttachedSource = source.attached(); + let path: String = attached + .source_file + .bind(py) + .getattr(RELATIVE_PATH_ATTRIBUTE)? + .str()? + .to_string(); + owners.push(resource_owner( + SOURCE_OWNER, + attached.name.clone(), + (path.clone(), path), + ( + attached.audits.bind(py).as_any(), + attached.columns.bind(py).as_any(), + ), + )?); + } + for seed in &self.seeds { + let (name, seed_file, schema_entry, schema_file) = seed.parts(); + let schema_entry: &Bound<'py, PyAny> = schema_entry.bind(py); + owners.push(resource_owner( + SEED_OWNER, + name.to_owned(), + ( + seed_file + .bind(py) + .getattr(RELATIVE_PATH_ATTRIBUTE)? + .str()? + .to_string(), + schema_file + .bind(py) + .getattr(RELATIVE_PATH_ATTRIBUTE)? + .str()? + .to_string(), + ), + ( + &schema_entry.getattr("audits")?, + &schema_entry.getattr("columns")?, + ), + )?); + } + Ok(owners) + } + + /// The kept model loop and its host. + fn attachment_loop(&self, py: Python<'_>) -> PyResult<(Py, Py)> { + match &self.attachment { + Some((model_loop, host)) => Ok((model_loop.clone_ref(py), host.clone_ref(py))), + None => Err(PyRuntimeError::new_err(ATTACHMENT_LOOP_UNSET)), + } + } + + /// Keep the model loop and its host for attaching the project's other resources. + pub(crate) fn keep_attachment_loop( + &mut self, + model_loop: Py, + host: Py, + ) { + self.attachment = Some((model_loop, host)); + } + + /// Every attached audit, in attachment order. + pub(crate) fn audits(&self) -> &[AuditRecord] { + &self.audits + } + + /// Every attached function, in attachment order. + pub(crate) fn functions(&self) -> &[FunctionRecord] { + &self.functions + } + + /// Every attached source, in attachment order. + pub(crate) fn sources(&self) -> &[SourceRecord] { + &self.sources + } + + /// Keep one attached source. + pub(crate) fn retain_source(&mut self, source: SourceRecord) { + self.sources.push(source); + } + + /// Every attached seed, in attachment order. + pub(crate) fn seeds(&self) -> &[SeedRecord] { + &self.seeds + } + + /// Every model's input record, in loop order. + pub(crate) fn inputs(&self) -> &[ModelInputRecord] { + &self.inputs + } + + /// The input record at `index`. + pub(crate) fn input(&self, index: usize) -> PyResult<&ModelInputRecord> { + self.inputs + .get(index) + .ok_or_else(|| PyIndexError::new_err(index)) + } + + /// The input record at `index`, for recording what later stages finish. + pub(crate) fn input_mut(&mut self, index: usize) -> PyResult<&mut ModelInputRecord> { + self.inputs + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index)) + } + + /// The classes model input views project their attributes with. + pub(crate) fn input_classes<'py>(&self, py: Python<'py>) -> PyResult> { + self.input_classes + .as_ref() + .map(|classes| classes.bind(py).clone()) + .ok_or_else(|| PyIndexError::new_err(INPUT_CLASSES_UNSET)) + } + + pub(crate) fn inferred_column<'py>( + &self, + py: Python<'py>, + classes: &CompiledViewClasses<'py>, + row: &InferredRow, + ) -> PyResult> { + let (name, column_type, nullability) = row; + if let Some(column) = self + .inferred_column_projections + .lock() + .map_err(|error| NativeCompilerError::new_err(error.to_string()))? + .get(row) + { + return Ok(column.bind(py).clone()); + } + let fields = PyDict::new(py); + fields.set_item("name", name)?; + fields.set_item("type", column_type.as_deref())?; + fields.set_item("nullability", classes.nullability.call1((nullability,))?)?; + let column = classes.inferred_column.call((), Some(&fields))?; + self.inferred_column_projections + .lock() + .map_err(|error| NativeCompilerError::new_err(error.to_string()))? + .insert(row.clone(), column.clone().unbind()); + Ok(column) + } +} + +#[pymethods] +impl NativeCompiledProject { + #[new] + fn new() -> Self { + Self::default() + } + + /// Retain one model's compile-input facts; a name recorded again is replaced in place. + fn record_model(&mut self, model: ModelInputPy<'_>) -> PyResult<()> { + compiler_guard(|| { + let facts: CompiledModelFacts = model_facts(model)?; + record_model(&mut self.facts, facts); + Ok(()) + }) + } + + /// Attach a recorded model's typed analysis; `False` when that model was never recorded. + fn record_model_analysis( + &mut self, + name: &str, + columns: AnalysisPy, + dynamic_proof: Option, + ) -> bool { + let (inferred, inferred_columns, nullable_columns, fast_lineage_has_star, unchecked) = + columns; + let analysis: ModelAnalysisFact = ModelAnalysisFact { + inferred_columns, + dynamic_proof: dynamic_proof.map( + |(output_proven, bare_dynamic_pivot, families, failure_reason, named_families)| { + DynamicProofFact { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + } + }, + ), + inferred, + nullable_columns, + fast_lineage_has_star, + unchecked_output_columns: unchecked, + }; + record_model_analysis(&mut self.facts, name, analysis) + } + + /// Record each assembled native model view's final typed analysis; return how many. + fn record_view_analyses( + slf: &Bound<'_, Self>, + views: Vec>, + ) -> PyResult { + let py: Python<'_> = slf.py(); + let mut facts: Vec<(String, ModelAnalysisFact)> = Vec::with_capacity(views.len()); + for view in &views { + facts.push(view.analysis_fact(py)?); + } + let mut owner = slf.borrow_mut(); + let mut recorded: usize = 0; + for (name, analysis) in facts { + recorded += usize::from(record_model_analysis(&mut owner.facts, &name, analysis)); + } + Ok(recorded) + } + + /// The compile inputs view of this owner's model inputs with these input fields. + fn inputs_view(slf: Bound<'_, Self>, fields: Bound<'_, PyDict>) -> PyResult { + NativeInputsView::new(slf.unbind(), &fields) + } + + /// The compiled project view of this owner's models with these project fields. + fn project_view( + slf: Bound<'_, Self>, + fields: Bound<'_, PyDict>, + ) -> PyResult { + NativeProjectView::new(slf.unbind(), &fields) + } + + /// Pair seed declarations with their CSV files and keep them; return each seed's view. + fn attach_seeds( + slf: &Bound<'_, Self>, + declarations: Vec<(Py, Py)>, + seed_files: Vec>, + compile_error: &Bound<'_, PyAny>, + ) -> PyResult> { + let seeds: Vec = paired_seeds(declarations, &seed_files, compile_error)?; + let count: usize = seeds.len(); + slf.borrow_mut().seeds = seeds; + Ok((0..count) + .map(|index| NativeSeedInput::new(slf.clone().unbind(), index)) + .collect()) + } + + /// Keep the destinations native project assembly resolved, in seed order. + fn record_seed_namespaces(&mut self, namespaces: Vec) -> PyResult<()> { + for (index, namespace) in namespaces.into_iter().enumerate() { + self.seeds + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_namespace(namespace); + } + Ok(()) + } + + /// Attach every discovered source in order with the kept loop; return each source's view. + fn attach_sources( + slf: &Bound<'_, Self>, + rows: Vec, + settings: SourceSettings, + ) -> PyResult> { + if rows.is_empty() { + return Ok(Vec::new()); + } + let py: Python<'_> = slf.py(); + let (model_loop, host) = slf.borrow().attachment_loop(py)?; + model_loop + .bind(py) + .get() + .run_sources(slf, host.bind(py), rows, settings) + } + + /// Attach every SQL, then Python, function with the kept loop; return each one's view. + fn attach_functions( + slf: &Bound<'_, Self>, + rows: (Vec, Vec), + settings: FunctionSettings, + ) -> PyResult> { + let py: Python<'_> = slf.py(); + let (model_loop, host) = slf.borrow().attachment_loop(py)?; + let models: Vec<(String, Vec)> = slf + .borrow() + .inputs + .iter() + .map(|record| { + Ok(( + record + .model_file + .bind(py) + .getattr(RELATIVE_PATH_ATTRIBUTE)? + .str()? + .to_string(), + record.references.clone(), + )) + }) + .collect::>()?; + let attached: Vec = + model_loop + .bind(py) + .get() + .run_functions(host.bind(py), rows, settings, &models)?; + let first: usize = slf.borrow().functions.len(); + let count: usize = attached.len(); + slf.borrow_mut() + .functions + .extend(attached.into_iter().map(FunctionRecord::new)); + Ok((first..first + count) + .map(|index| NativeFunctionInput::new(slf.clone().unbind(), index)) + .collect()) + } + + /// The orphaned audit factory warnings, missing descriptions and cursor-input errors. + fn resource_diagnostics<'py>( + &self, + py: Python<'py>, + settings: DiagnosticSettings, + ) -> PyResult> { + resource_diagnostics(self, py, &settings) + } + + /// Attach every SQL test case, then every scenario, with the kept loop. + fn attach_sql_tests( + slf: &Bound<'_, Self>, + files: Vec, + scenarios: Vec, + settings: SqlTestSettings, + ) -> PyResult { + let py: Python<'_> = slf.py(); + let (model_loop, host) = slf.borrow().attachment_loop(py)?; + let arities: HashMap = slf + .borrow() + .functions + .iter() + .filter(|record| !record.attached.return_columns.is_empty()) + .map(|record| { + ( + record.attached.name.clone(), + record.attached.arguments.len(), + ) + }) + .collect(); + model_loop.bind(py).get().run_sql_tests( + host.bind(py), + (&files, &scenarios), + &settings, + arities, + ) + } + + /// Attach singular, then resource-attached, audits; return views and override warnings. + fn attach_audits<'py>( + slf: &Bound<'py, Self>, + files: Vec, + settings: AuditSettings, + ) -> PyResult<(Vec, Vec>)> { + let py: Python<'py> = slf.py(); + let (model_loop, host) = slf.borrow().attachment_loop(py)?; + let owners: Vec> = slf.borrow().attachment_owners(py)?; + let (attached, shadows) = + model_loop + .bind(py) + .get() + .run_audits(host.bind(py), &files, owners, &settings)?; + let first: usize = slf.borrow().audits.len(); + let count: usize = attached.len(); + slf.borrow_mut() + .audits + .extend(attached.into_iter().map(AuditRecord::new)); + Ok(( + (first..first + count) + .map(|index| NativeAuditInput::new(slf.clone().unbind(), index)) + .collect(), + shadows, + )) + } + + /// Keep the scope dependency keys native project assembly resolved, in audit order. + fn record_audit_scope_deps(&mut self, deps: Vec>) -> PyResult<()> { + for (index, keys) in deps.into_iter().enumerate() { + self.audits + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_scope_deps(keys); + } + Ok(()) + } + + /// The assembled audit at `index`. + fn compiled_audit(slf: &Bound<'_, Self>, index: usize) -> PyResult { + if index >= slf.borrow().audits.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledAudit::new(slf.clone().unbind(), index)) + } + + /// Keep the dependency keys native project assembly resolved, in function order. + fn record_function_deps(&mut self, deps: Vec>) -> PyResult<()> { + for (index, keys) in deps.into_iter().enumerate() { + self.functions + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_deps(keys); + } + Ok(()) + } + + /// The assembled function at `index`. + fn compiled_function(slf: &Bound<'_, Self>, index: usize) -> PyResult { + if index >= slf.borrow().functions.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledFunction::new(slf.clone().unbind(), index)) + } + + /// Keep the namespaces native project assembly resolved, in source order. + fn record_source_namespaces(&mut self, namespaces: Vec) -> PyResult<()> { + for (index, namespace) in namespaces.into_iter().enumerate() { + self.sources + .get_mut(index) + .ok_or_else(|| PyIndexError::new_err(index))? + .set_namespace(namespace); + } + Ok(()) + } + + /// The assembled source at `index`. + fn compiled_source(slf: &Bound<'_, Self>, index: usize) -> PyResult { + if index >= slf.borrow().sources.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledSource::new(slf.clone().unbind(), index)) + } + + /// The assembled seed at `index`. + fn compiled_seed(slf: &Bound<'_, Self>, index: usize) -> PyResult { + if index >= slf.borrow().seeds.len() { + return Err(PyIndexError::new_err(index)); + } + Ok(NativeCompiledSeed::new(slf.clone().unbind(), index)) + } + + /// How many lineage graphs this owner's project views built. + fn graph_builds(&self) -> usize { + self.graph_builds + } + + /// Whether both projects retained the same facts. + fn __eq__(&self, other: &Self) -> bool { + self.facts == other.facts + } + + /// The recorded models' names, in production order. + fn model_names(&self) -> Vec { + project_models(&self.facts) + .iter() + .map(|model| model.name.clone()) + .collect() + } +} + +pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult { + let ModelInputPy( + (name, relative_path, query_sql, authored_query_sql, authored_sql), + (values, header_keys, layer_schema), + references, + schema, + enum_columns, + (enum_declarations, constant_declarations), + ) = model; + let config_values: ConfigValues = config_values(&values); + Ok(CompiledModelFacts { + name, + relative_path, + query_sql, + authored_query_sql, + authored_sql, + config: ModelConfigFacts { + values: config_values.0, + encoding_error: config_values.1, + header_keys, + layer_schema, + contract: text_value(&values, CONTRACT_KEY)?, + materialized: materialization(&values)?, + incremental_mode: text_value(&values, INCREMENTAL_MODE_KEY)?, + }, + references: references + .into_iter() + .map(|(kind, name, package)| ModelReferenceFact { + kind, + name, + package, + }) + .collect(), + schema: schema.map(schema_fact), + enum_columns, + enum_declarations: declarations(enum_declarations)?, + constant_declarations: declarations(constant_declarations)?, + analysis: None, + }) +} + +/// The config values, or the encoder's error, kept for the consumer that reads them. +pub(crate) fn config_values(values: &Bound<'_, PyDict>) -> ConfigValues { + match object_value(values) { + Ok(Value::Object(entries)) => (entries.into_iter().collect(), None), + Ok(_) => (BTreeMap::new(), None), + Err(error) => (BTreeMap::new(), Some(error.to_string())), + } +} + +/// The `materialized` config value when it is text, `table` when it is absent. +pub(crate) fn materialization(values: &Bound<'_, PyDict>) -> PyResult> { + if values.get_item(MATERIALIZED_KEY)?.is_none() { + return Ok(Some(TABLE_MATERIALIZATION.to_owned())); + } + text_value(values, MATERIALIZED_KEY) +} + +/// A config value when it is text. +pub(crate) fn text_value(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match values.get_item(key)? { + Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} + +pub(crate) fn schema_fact( + (audit_count, columns, dynamic_columns, (type_enforcement, named_schema)): SchemaPy, +) -> ModelSchemaFact { + ModelSchemaFact { + audit_count, + columns: columns + .into_iter() + .map( + |(name, data_type, nullable, audit_count, declared_outside_model)| { + SchemaColumnFact { + name, + data_type, + nullable, + audit_count, + declared_outside_model, + } + }, + ) + .collect(), + dynamic_columns: dynamic_columns.into_iter().map(family_fact).collect(), + type_enforcement, + named_schema, + } +} + +fn family_fact( + (key, name, pivot_column, value_column, aggregate, data_type, name_pattern): DynamicFamilyPy, +) -> DynamicFamilyFact { + DynamicFamilyFact { + key, + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + } +} + +pub(crate) fn declarations(rows: Vec>) -> PyResult> { + rows.into_iter().map(declaration).collect() +} + +fn declaration(row: DeclarationPy<'_>) -> PyResult { + let DeclarationPy(name, relative_path, members, value, value_type, render_as) = row; + Ok(DeclarationFact { + name, + relative_path, + members: members + .into_iter() + .map(|(name, value)| Ok((name, plain_value(&value)?))) + .collect::>()?, + value: Some(plain_value(&value)?).filter(|value| !value.is_null()), + value_type, + render_as, + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs new file mode 100644 index 0000000000..8d7cfae0eb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -0,0 +1,549 @@ +//! A compiled project, read from the native compiled project that owns its models. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::{pyclass, pymethods}; + +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + NativeCompiledModel, model_graph_row, +}; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + InputViewClasses, authored_expansion, view_classes, +}; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::graph::project_graph::{ + NativeProjectGraph, ResourceRow, project_graph, +}; + +const MODELS_FIELD: &str = "models"; +const RUN_ID_FIELD: &str = "run_id"; +const ENFORCE_EXPLICIT_FIELD: &str = "enforce_explicit_references"; +const FILE_PATH_ATTRIBUTE: &str = "file_path"; +const KEY_ATTRIBUTE: &str = "key"; +const DEPS_ATTRIBUTE: &str = "deps"; +const TAGS_ATTRIBUTE: &str = "tags"; +const SCHEMA_ENTRY_ATTRIBUTE: &str = "schema_entry"; +const RESOURCE_TYPE_ATTRIBUTE: &str = "resource_type"; +const NAME_ATTRIBUTE: &str = "name"; + +/// Project inputs held as Python objects: user code, discovery or an integration owns them. +struct BoundaryInputs { + /// The resolved connection, adapter integration values. + effective_connection: Py, + /// Project variables, user values. + effective_vars: Py, + /// The native binding catalog. + binding_catalog: Py, + /// The adapter's lexical syntax. + sql_lexical_syntax: Py, + /// The compile cache directory path. + compile_cache_dir: Py, + /// Discovered loader functions, user callables. + loader_functions: Py, + /// Discovered Python hooks, user callables. + hook_functions: Py, + /// Discovered SQL hook files. + sql_hook_files: Py, + /// Discovered custom materializations. + materialization_files: Py, + /// Public enums, user declaration values. + public_enums: Py, + /// Public constants, user declaration values. + public_constants: Py, + /// Loaded macro modules, user code. + loaded_macros: Py, + /// The external reference resolver integration, or `None`. + external_sql_reference_resolver: Py, + /// The native analysis session, or `None`. + native_session: Py, +} + +/// Compiler-derived project parts still held as Python objects until they are owner facts. +struct CompilerHandles { + /// Effective settings. + settings: Py, + /// Effective scenario config. + scenario: Py, + /// Compiled sources. + sources: Py, + /// Compiled seeds. + seeds: Py, + /// Compiled SQL functions. + functions: Py, + /// Compiled audits. + audits: Py, + /// Compiled SQL tests. + sql_tests: Py, + /// Compiled SQL scenarios. + sql_scenarios: Py, + /// Compiler diagnostics. + diagnostics: Py, + /// Literal relations no resource matched. + unmatched_literal_sql_relations: Py, + /// The declaration scope index. + scope_index: Py, +} + +/// The project-level values of one compiled project view. +struct ProjectFields { + run_id: String, + effective_target_name: Option, + effective_target_database: Option, + effective_target_schema: Option, + sql_analysis_dialect: Option, + enforce_explicit_references: bool, + models: Py, + boundary: BoundaryInputs, + compiler: CompilerHandles, +} + +impl ProjectFields { + /// The fields named in `fields`, each required. + fn from_fields(fields: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + run_id: required(fields, RUN_ID_FIELD)?.extract()?, + effective_target_name: required(fields, "effective_target_name")?.extract()?, + effective_target_database: required(fields, "effective_target_database")?.extract()?, + effective_target_schema: required(fields, "effective_target_schema")?.extract()?, + sql_analysis_dialect: required(fields, "sql_analysis_dialect")?.extract()?, + enforce_explicit_references: required(fields, ENFORCE_EXPLICIT_FIELD)?.extract()?, + models: required(fields, MODELS_FIELD)? + .downcast_into::()? + .unbind(), + boundary: BoundaryInputs { + effective_connection: required(fields, "effective_connection")?.unbind(), + effective_vars: required(fields, "effective_vars")?.unbind(), + binding_catalog: required(fields, "binding_catalog")?.unbind(), + sql_lexical_syntax: required(fields, "sql_lexical_syntax")?.unbind(), + compile_cache_dir: required(fields, "compile_cache_dir")?.unbind(), + loader_functions: required(fields, "loader_functions")?.unbind(), + hook_functions: required(fields, "hook_functions")?.unbind(), + sql_hook_files: required(fields, "sql_hook_files")?.unbind(), + materialization_files: required(fields, "materialization_files")?.unbind(), + public_enums: required(fields, "public_enums")?.unbind(), + public_constants: required(fields, "public_constants")?.unbind(), + loaded_macros: required(fields, "loaded_macros")?.unbind(), + external_sql_reference_resolver: required( + fields, + "external_sql_reference_resolver", + )? + .unbind(), + native_session: required(fields, "native_session")?.unbind(), + }, + compiler: CompilerHandles { + settings: required(fields, "settings")?.unbind(), + scenario: required(fields, "scenario")?.unbind(), + sources: required(fields, "sources")?.unbind(), + seeds: required(fields, "seeds")?.unbind(), + functions: required(fields, "functions")?.unbind(), + audits: required(fields, "audits")?.unbind(), + sql_tests: required(fields, "sql_tests")?.unbind(), + sql_scenarios: required(fields, "sql_scenarios")?.unbind(), + diagnostics: required(fields, "diagnostics")?.unbind(), + unmatched_literal_sql_relations: required( + fields, + "unmatched_literal_sql_relations", + )? + .unbind(), + scope_index: required(fields, "scope_index")?.unbind(), + }, + }) + } + + /// These fields with `changes` applied. + fn changed(&self, py: Python<'_>, changes: &Bound<'_, PyDict>) -> PyResult { + let current: Bound<'_, PyDict> = self.as_fields(py)?; + for (name, value) in changes.iter() { + if !current.contains(&name)? { + return Err(PyTypeError::new_err(format!( + "unknown project field {name}" + ))); + } + current.set_item(name, value)?; + } + Self::from_fields(¤t) + } + + /// Every field by name. + fn as_fields<'py>(&self, py: Python<'py>) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item(RUN_ID_FIELD, &self.run_id)?; + fields.set_item("effective_target_name", &self.effective_target_name)?; + fields.set_item("effective_target_database", &self.effective_target_database)?; + fields.set_item("effective_target_schema", &self.effective_target_schema)?; + fields.set_item("sql_analysis_dialect", &self.sql_analysis_dialect)?; + fields.set_item(ENFORCE_EXPLICIT_FIELD, self.enforce_explicit_references)?; + fields.set_item(MODELS_FIELD, self.models.bind(py))?; + fields.set_item( + "effective_connection", + self.boundary.effective_connection.bind(py), + )?; + fields.set_item("effective_vars", self.boundary.effective_vars.bind(py))?; + fields.set_item("binding_catalog", self.boundary.binding_catalog.bind(py))?; + fields.set_item( + "sql_lexical_syntax", + self.boundary.sql_lexical_syntax.bind(py), + )?; + fields.set_item( + "compile_cache_dir", + self.boundary.compile_cache_dir.bind(py), + )?; + fields.set_item("loader_functions", self.boundary.loader_functions.bind(py))?; + fields.set_item("hook_functions", self.boundary.hook_functions.bind(py))?; + fields.set_item("sql_hook_files", self.boundary.sql_hook_files.bind(py))?; + fields.set_item( + "materialization_files", + self.boundary.materialization_files.bind(py), + )?; + fields.set_item("public_enums", self.boundary.public_enums.bind(py))?; + fields.set_item("public_constants", self.boundary.public_constants.bind(py))?; + fields.set_item("loaded_macros", self.boundary.loaded_macros.bind(py))?; + fields.set_item( + "external_sql_reference_resolver", + self.boundary.external_sql_reference_resolver.bind(py), + )?; + fields.set_item("native_session", self.boundary.native_session.bind(py))?; + fields.set_item("settings", self.compiler.settings.bind(py))?; + fields.set_item("scenario", self.compiler.scenario.bind(py))?; + fields.set_item("sources", self.compiler.sources.bind(py))?; + fields.set_item("seeds", self.compiler.seeds.bind(py))?; + fields.set_item("functions", self.compiler.functions.bind(py))?; + fields.set_item("audits", self.compiler.audits.bind(py))?; + fields.set_item("sql_tests", self.compiler.sql_tests.bind(py))?; + fields.set_item("sql_scenarios", self.compiler.sql_scenarios.bind(py))?; + fields.set_item("diagnostics", self.compiler.diagnostics.bind(py))?; + fields.set_item( + "unmatched_literal_sql_relations", + self.compiler.unmatched_literal_sql_relations.bind(py), + )?; + fields.set_item("scope_index", self.compiler.scope_index.bind(py))?; + Ok(fields) + } +} + +fn required<'py>(fields: &Bound<'py, PyDict>, name: &str) -> PyResult> { + fields + .get_item(name)? + .ok_or_else(|| PyTypeError::new_err(format!("missing project field {name}"))) +} + +/// One compiled project, its models projected from the owner. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeProjectView { + owner: Py, + fields: ProjectFields, + lineage_graph: PyOnceLock>, + sql_expansions: PyOnceLock>, +} + +impl NativeProjectView { + /// The view over `owner` with these project fields. + pub(crate) fn new( + owner: Py, + fields: &Bound<'_, PyDict>, + ) -> PyResult { + Ok(Self::with_fields( + owner, + ProjectFields::from_fields(fields)?, + )) + } + + fn with_fields(owner: Py, fields: ProjectFields) -> Self { + Self { + owner, + fields, + lineage_graph: PyOnceLock::new(), + sql_expansions: PyOnceLock::new(), + } + } +} + +#[pymethods] +impl NativeProjectView { + /// This project with `changes` applied; the owner's models are shared. + #[pyo3(signature = (**changes))] + fn with_changes(&self, py: Python<'_>, changes: Option>) -> PyResult { + let fields: ProjectFields = match changes { + Some(changes) => self.fields.changed(py, &changes)?, + None => self.fields.changed(py, &PyDict::new(py))?, + }; + Ok(Self::with_fields(self.owner.clone_ref(py), fields)) + } + + /// Every field by name, for an explicit `CompiledProject` projection. + #[pyo3(signature = (include_sql_expansions = true))] + fn fields<'py>( + &self, + py: Python<'py>, + include_sql_expansions: bool, + ) -> PyResult> { + let fields: Bound<'py, PyDict> = self.fields.as_fields(py)?; + fields.set_item( + "sql_expansions", + if include_sql_expansions { + self.sql_expansions(py)? + } else { + PyDict::new(py).unbind() + }, + )?; + fields.set_item("native_project", self.owner.bind(py))?; + Ok(fields) + } + + #[getter] + fn run_id(&self) -> String { + self.fields.run_id.clone() + } + + #[getter] + fn effective_target_name(&self) -> Option { + self.fields.effective_target_name.clone() + } + + #[getter] + fn effective_target_database(&self) -> Option { + self.fields.effective_target_database.clone() + } + + #[getter] + fn effective_target_schema(&self) -> Option { + self.fields.effective_target_schema.clone() + } + + #[getter] + fn sql_analysis_dialect(&self) -> Option { + self.fields.sql_analysis_dialect.clone() + } + + #[getter(enforce_explicit_references)] + fn is_explicit_reference_enforced(&self) -> bool { + self.fields.enforce_explicit_references + } + + #[getter] + fn models(&self, py: Python<'_>) -> Py { + self.fields.models.clone_ref(py) + } + + #[getter] + fn native_project(&self, py: Python<'_>) -> Py { + self.owner.clone_ref(py) + } + + #[getter] + fn effective_connection(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_connection.clone_ref(py) + } + + #[getter] + fn effective_vars(&self, py: Python<'_>) -> Py { + self.fields.boundary.effective_vars.clone_ref(py) + } + + #[getter] + fn binding_catalog(&self, py: Python<'_>) -> Py { + self.fields.boundary.binding_catalog.clone_ref(py) + } + + #[getter] + fn sql_lexical_syntax(&self, py: Python<'_>) -> Py { + self.fields.boundary.sql_lexical_syntax.clone_ref(py) + } + + #[getter] + fn compile_cache_dir(&self, py: Python<'_>) -> Py { + self.fields.boundary.compile_cache_dir.clone_ref(py) + } + + #[getter] + fn loader_functions(&self, py: Python<'_>) -> Py { + self.fields.boundary.loader_functions.clone_ref(py) + } + + #[getter] + fn hook_functions(&self, py: Python<'_>) -> Py { + self.fields.boundary.hook_functions.clone_ref(py) + } + + #[getter] + fn sql_hook_files(&self, py: Python<'_>) -> Py { + self.fields.boundary.sql_hook_files.clone_ref(py) + } + + #[getter] + fn materialization_files(&self, py: Python<'_>) -> Py { + self.fields.boundary.materialization_files.clone_ref(py) + } + + #[getter] + fn public_enums(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_enums.clone_ref(py) + } + + #[getter] + fn public_constants(&self, py: Python<'_>) -> Py { + self.fields.boundary.public_constants.clone_ref(py) + } + + #[getter] + fn loaded_macros(&self, py: Python<'_>) -> Py { + self.fields.boundary.loaded_macros.clone_ref(py) + } + + #[getter] + fn external_sql_reference_resolver(&self, py: Python<'_>) -> Py { + self.fields + .boundary + .external_sql_reference_resolver + .clone_ref(py) + } + + #[getter] + fn native_session(&self, py: Python<'_>) -> Py { + self.fields.boundary.native_session.clone_ref(py) + } + + #[getter] + fn settings(&self, py: Python<'_>) -> Py { + self.fields.compiler.settings.clone_ref(py) + } + + #[getter] + fn scenario(&self, py: Python<'_>) -> Py { + self.fields.compiler.scenario.clone_ref(py) + } + + #[getter] + fn sources(&self, py: Python<'_>) -> Py { + self.fields.compiler.sources.clone_ref(py) + } + + #[getter] + fn seeds(&self, py: Python<'_>) -> Py { + self.fields.compiler.seeds.clone_ref(py) + } + + #[getter] + fn functions(&self, py: Python<'_>) -> Py { + self.fields.compiler.functions.clone_ref(py) + } + + #[getter] + fn audits(&self, py: Python<'_>) -> Py { + self.fields.compiler.audits.clone_ref(py) + } + + #[getter] + fn sql_tests(&self, py: Python<'_>) -> Py { + self.fields.compiler.sql_tests.clone_ref(py) + } + + #[getter] + fn sql_scenarios(&self, py: Python<'_>) -> Py { + self.fields.compiler.sql_scenarios.clone_ref(py) + } + + #[getter] + fn diagnostics(&self, py: Python<'_>) -> Py { + self.fields.compiler.diagnostics.clone_ref(py) + } + + #[getter] + fn unmatched_literal_sql_relations(&self, py: Python<'_>) -> Py { + self.fields + .compiler + .unmatched_literal_sql_relations + .clone_ref(py) + } + + #[getter] + fn scope_index(&self, py: Python<'_>) -> Py { + self.fields.compiler.scope_index.clone_ref(py) + } + + /// Each model's authored-to-expanded passes by file path, where interpolation kept the query. + #[getter] + fn sql_expansions(&self, py: Python<'_>) -> PyResult> { + self.sql_expansions + .get_or_try_init(py, || { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + let owner = self.owner.bind(py).borrow(); + let expansions: Bound<'_, PyDict> = PyDict::new(py); + for record in owner.inputs() { + let expansion: Py = authored_expansion(&classes, record)?; + if !expansion.is_none(py) { + let path: Bound<'_, PyAny> = + record.model_file.bind(py).getattr(FILE_PATH_ATTRIBUTE)?; + expansions.set_item(path, expansion)?; + } + } + Ok::<_, pyo3::PyErr>(expansions.unbind()) + }) + .map(|expansions| expansions.clone_ref(py)) + } + + /// The lineage edges, tags, model folders and names, indexed natively once per project. + #[getter] + fn lineage_graph(&self, py: Python<'_>) -> PyResult> { + self.lineage_graph + .get_or_try_init(py, || { + let mut rows: Vec = Vec::new(); + for model in self.fields.models.bind(py).iter() { + let model: Bound<'_, NativeCompiledModel> = model.downcast_into()?; + rows.push(model_graph_row(model.get(), py)?); + } + let compiler: &CompilerHandles = &self.fields.compiler; + for source in compiler.sources.bind(py).try_iter()? { + rows.push(resource_row(&source?, None)?); + } + for seed in compiler.seeds.bind(py).try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + let tags: Bound<'_, PyAny> = seed + .getattr(SCHEMA_ENTRY_ATTRIBUTE)? + .getattr(TAGS_ATTRIBUTE)?; + rows.push(resource_row(&seed, Some(&tags))?); + } + for function in compiler.functions.bind(py).try_iter()? { + let function: Bound<'_, PyAny> = function?; + let tags: Bound<'_, PyAny> = function.getattr(TAGS_ATTRIBUTE)?; + rows.push(resource_row(&function, Some(&tags))?); + } + self.owner.bind(py).borrow_mut().record_graph_build(); + Py::new(py, project_graph(rows)?) + }) + .map(|graph| graph.clone_ref(py)) + } +} + +/// A source, seed or function's graph row, read from its compiled object. +fn resource_row( + resource: &Bound<'_, PyAny>, + tags: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let mut deps: Vec<(String, String)> = Vec::new(); + for key in resource.getattr(DEPS_ATTRIBUTE)?.try_iter()? { + deps.push(key_row(&key?)?); + } + let mut tag_names: Vec = Vec::new(); + if let Some(tags) = tags { + for tag in tags.try_iter()? { + tag_names.push(tag?.str()?.extract()?); + } + } + Ok(( + key_row(&resource.getattr(KEY_ATTRIBUTE)?)?, + deps, + tag_names, + None, + )) +} + +/// `(resource type, name)` of a `CompiledObjectKey`. +pub(crate) fn key_row(key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + Ok(( + key.getattr(RESOURCE_TYPE_ATTRIBUTE)?.str()?.extract()?, + key.getattr(NAME_ATTRIBUTE)?.extract()?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs new file mode 100644 index 0000000000..4b36f9b0eb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -0,0 +1,6 @@ +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + crate::bindings::_helpers::compiled_project::project::register(module)?; + crate::bindings::_helpers::compiled_project::model_loop::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs new file mode 100644 index 0000000000..d500e6fba9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -0,0 +1,527 @@ +//! Compile-input diagnostics of the project's resources, read from the native owner. + +use pyo3::exceptions::PyValueError; +use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; +use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods}; +use sqlbuild_analysis::compiled_project::main::text_position::text_position; +use sqlbuild_analysis::compiled_project::models::TextPosition; +use sqlbuild_analysis::contracts::main::join_helps::join_helps; +use sqlbuild_analysis::contracts::main::snippet_help::snippet_help; +use sqlbuild_analysis::resource_diagnostics::main::cursor_lacks_inputs::cursor_lacks_inputs; +use sqlbuild_analysis::resource_diagnostics::main::header_code_offset::header_code_offset; +use sqlbuild_analysis::resource_diagnostics::main::header_help::header_help; +use sqlbuild_analysis::resource_diagnostics::main::missing_description::missing_description; +use sqlbuild_analysis::resource_diagnostics::main::yaml_help::yaml_help; +use sqlbuild_analysis::resource_diagnostics::models::{CursorModel, HeaderHelp}; + +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; + +const MISSING_DESCRIPTION_CODE: &str = "P010"; +const CURSOR_WITHOUT_INPUTS_CODE: &str = "P011"; +const ORPHAN_FACTORY_CODE: &str = "C216"; +const AUDIT_FACTORIES_KEY: &str = "audit_factories"; +const CURSOR_KEY: &str = "cursor"; +const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; +const PYTHON_UDF_DECORATOR: &str = "udf"; + +/// A loader's description, relative path and whether that path is a YAML file. +type LoaderRow = (Option, String, bool); + +/// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. +type PythonResourceRow = (String, String, Option>, Py, String); + +/// What the resource diagnostics read beyond the owner, and the Python values they build. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct DiagnosticSettings { + /// `CompilerDiagnostic`, `SourceLocation`, `Path` and `CompileInputError`. + compiler_diagnostic: Py, + compile_error: Py, + source_location: Py, + path_class: Py, + /// Every discovered model file, whose headers attach audit factories. + model_files: Vec>, + /// Each discovered audit factory's name, relative path and line. + audit_factories: Vec<(String, String, usize)>, + /// Each scenario's name, relative path, contents and authored header description. + scenarios: Vec<(String, String, String, Py)>, + /// Each SQL hook file's name, relative path, contents and description. + sql_hooks: Vec<(String, String, String, Option)>, + /// Each loader by name. + loaders: HashMap, + /// Python-defined resources: kind, name, description, location and help, read by Python. + python_resources: Vec, + /// The Python locators module: YAML entry lines, MODEL header spans and decorators. + locators: Py, +} + +/// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. +pub(crate) type ResourceDiagnostics<'py> = ( + Vec>, + Vec>, + Vec>, +); + +struct Diagnostics<'a, 'py> { + py: Python<'py>, + settings: &'a DiagnosticSettings, + yaml_lines: RefCell>>, +} + +impl<'py> Diagnostics<'_, 'py> { + fn path(&self, relative_path: &str) -> PyResult> { + self.settings + .path_class + .bind(self.py) + .call1((relative_path,)) + } + + fn location( + &self, + relative_path: &str, + position: (usize, usize), + end: Option<(usize, usize)>, + ) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(self.py); + fields.set_item("path", self.path(relative_path)?)?; + fields.set_item("line", position.0)?; + fields.set_item("column", position.1)?; + if let Some((end_line, end_column)) = end { + fields.set_item("end_line", end_line)?; + fields.set_item("end_column", end_column)?; + } + self.settings + .source_location + .bind(self.py) + .call((), Some(&fields)) + } + + /// The first code after leading whitespace and comments, raising Python's comment error. + fn header_location(&self, relative_path: &str, contents: &str) -> PyResult> { + let offset: usize = header_code_offset(contents).map_err(|message| { + match self.settings.compile_error.bind(self.py).call1((message,)) { + Ok(error) => pyo3::PyErr::from_value(error), + Err(failure) => failure, + } + })?; + let position: TextPosition = text_position(contents, offset); + self.location(relative_path, (position.line, position.column), None) + } + + fn yaml_location( + &self, + relative_path: &str, + contents: &str, + name: &str, + ) -> PyResult> { + let line: usize = { + let mut files = self.yaml_lines.borrow_mut(); + let entries = files.entry(relative_path.to_owned()).or_insert_with(|| { + let mut entries: HashMap = HashMap::new(); + for (entry, line) in yaml_entry_lines(contents) { + entries.entry(entry).or_insert(line); + } + entries + }); + entries.get(name).copied().unwrap_or(1) + }; + self.location(relative_path, (line, 1), None) + } + + /// One diagnostic: `(code, severity)`, `(message, help)`, the resource and its location. + fn diagnostic( + &self, + code: (&str, &str), + text: (&str, Option<&str>), + resource: (Option<&str>, &str), + location: Bound<'py, PyAny>, + ) -> PyResult> { + let (code, severity) = code; + let (message, help) = text; + let (resource_type, resource_name) = resource; + let fields: Bound<'py, PyDict> = PyDict::new(self.py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", severity)?; + fields.set_item("code", code)?; + fields.set_item("message", message)?; + fields.set_item("resource_type", resource_type)?; + fields.set_item("resource_name", resource_name)?; + fields.set_item("location", location)?; + fields.set_item("help", help)?; + self.settings + .compiler_diagnostic + .bind(self.py) + .call((), Some(&fields)) + } + + /// P010 for `kind` `name` when its description is missing or blank. + fn described( + &self, + resource: (&str, &str, Option<&str>), + description: Option<&str>, + location: impl FnOnce() -> PyResult>, + help: &str, + ) -> PyResult>> { + let (kind, name, resource_type) = resource; + let Some(message) = missing_description(kind, name, description) else { + return Ok(None); + }; + Ok(Some(self.diagnostic( + (MISSING_DESCRIPTION_CODE, "error"), + (&message, Some(help)), + (resource_type, name), + location()?, + )?)) + } +} + +/// Orphaned factory warnings, missing descriptions and cursor errors of the owner's resources. +pub(crate) fn resource_diagnostics<'py>( + project: &NativeCompiledProject, + py: Python<'py>, + settings: &DiagnosticSettings, +) -> PyResult> { + let diagnostics = Diagnostics { + py, + settings, + yaml_lines: RefCell::new(HashMap::new()), + }; + Ok(( + orphaned_factories(&diagnostics)?, + missing_descriptions(project, &diagnostics)?, + cursor_without_inputs(project, &diagnostics)?, + )) +} + +fn orphaned_factories<'py>(diagnostics: &Diagnostics<'_, 'py>) -> PyResult>> { + let py: Python<'py> = diagnostics.py; + let mut referenced: HashSet = HashSet::new(); + for model_file in &diagnostics.settings.model_files { + let header: Bound<'py, PyDict> = model_file + .bind(py) + .getattr("header_values")? + .downcast_into()?; + let Some(names) = header.get_item(AUDIT_FACTORIES_KEY)? else { + continue; + }; + let Ok(names) = names.downcast::() else { + continue; + }; + for name in names.iter() { + if name.is_instance_of::() { + referenced.insert(name.extract()?); + } + } + } + let mut warnings: Vec> = Vec::new(); + for (name, relative_path, line) in &diagnostics.settings.audit_factories { + if referenced.contains(name) { + continue; + } + warnings.push(diagnostics.diagnostic( + (ORPHAN_FACTORY_CODE, "warning"), + ( + &format!( + "Audit factory '{name}' is not referenced by any model and will not produce \ + audits" + ), + None, + ), + (Some("audit"), name), + diagnostics.location(relative_path, (*line, 1), None)?, + )?); + } + Ok(warnings) +} + +fn posix(path: &str) -> String { + path.replace('\\', "/") +} + +fn missing_descriptions<'py>( + project: &NativeCompiledProject, + diagnostics: &Diagnostics<'_, 'py>, +) -> PyResult>> { + let py: Python<'py> = diagnostics.py; + let mut found: Vec>> = Vec::new(); + for record in project.inputs() { + let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); + let name: String = model_file + .getattr("file_path")? + .getattr("stem")? + .extract()?; + let path: String = model_file.getattr("relative_path")?.str()?.extract()?; + let contents: String = model_file.getattr("contents")?.extract()?; + let entry: String = format!("description \"What one row of {name} represents\""); + let help: String = header_help(&HeaderHelp { + kind: "model", + name: &name, + statement: "MODEL", + entry: &entry, + path: &posix(&path), + description_only: false, + }); + let description: Option<&str> = record + .schema + .as_ref() + .and_then(|schema| schema.description.as_deref()); + found.push(diagnostics.described( + ("model", &name, Some("model")), + description, + || diagnostics.header_location(&path, &contents), + &help, + )?); + } + for (name, path, contents, raw) in &diagnostics.settings.scenarios { + let raw: &Bound<'py, PyAny> = raw.bind(py); + let description: Option = if raw.is_instance_of::() { + Some(raw.extract()?) + } else { + None + }; + let entry: String = format!("description \"The behaviour {name} proves\""); + let help: String = header_help(&HeaderHelp { + kind: "scenario", + name, + statement: "SCENARIO", + entry: &entry, + path: &posix(path), + description_only: false, + }); + found.push(diagnostics.described( + ("scenario", name, Some("sql_scenario")), + description.as_deref(), + || diagnostics.header_location(path, contents), + &help, + )?); + } + for seed in project.seeds() { + let (_, _, entry, schema_file) = seed.parts(); + let entry: &Bound<'py, PyAny> = entry.bind(py); + let schema_file: &Bound<'py, PyAny> = schema_file.bind(py); + let name: String = entry.getattr("name")?.extract()?; + let description: Option = entry.getattr("description")?.extract()?; + let path: String = schema_file.getattr("relative_path")?.str()?.extract()?; + let contents_value = schema_file.getattr("contents")?; + let contents = contents_value.downcast::()?.to_str()?; + found.push(diagnostics.described( + ("seed", &name, Some("seed")), + description.as_deref(), + || diagnostics.yaml_location(&path, contents, &name), + &yaml_help("seed", &name, &posix(&path)), + )?); + } + for source in project.sources() { + found.push(source_description(diagnostics, source)?); + } + for function in project.functions() { + found.push(function_description(diagnostics, function)?); + } + for (name, path, contents, description) in &diagnostics.settings.sql_hooks { + let entry: String = format!("description \"What {name} does\""); + let help: String = header_help(&HeaderHelp { + kind: "hook", + name, + statement: "HOOK", + entry: &entry, + path: &posix(path), + description_only: true, + }); + found.push(diagnostics.described( + ("hook", name, None), + description.as_deref(), + || diagnostics.header_location(path, contents), + &help, + )?); + } + for (kind, name, description, location, help) in &diagnostics.settings.python_resources { + let text = description + .as_ref() + .map(|description| docstring_text(description.bind(py))); + found.push(diagnostics.described( + (kind, name, None), + text.as_deref(), + || location.bind(py).call0(), + help, + )?); + } + Ok(found.into_iter().flatten().collect()) +} + +/// A docstring's text; a lone surrogate becomes U+FFFD, which `str.strip` keeps as it would. +fn docstring_text(description: &Bound<'_, PyString>) -> String { + description.to_string_lossy().into_owned() +} + +/// A source's P010, falling back to its loader's description and naming the loader in help. +fn source_description<'py>( + diagnostics: &Diagnostics<'_, 'py>, + source: &crate::bindings::_helpers::compiled_project::source_views::SourceRecord, +) -> PyResult>> { + let py: Python<'py> = diagnostics.py; + let attached = source.attached(); + let name: &str = &attached.name; + let source_file: &Bound<'py, PyAny> = attached.source_file.bind(py); + let path: String = source_file.getattr("relative_path")?.str()?.extract()?; + let contents_value = source_file.getattr("contents")?; + let contents = contents_value.downcast::()?.to_str()?; + let loader_name: Option = attached.entry.bind(py).getattr("loader")?.extract()?; + let loader: Option<(&String, &LoaderRow)> = loader_name + .as_ref() + .and_then(|loader| diagnostics.settings.loaders.get_key_value(loader)); + let mut description: Option = attached.description.clone(); + if let Some((_, (loader_description, _, _))) = loader + && missing_description("", "", description.as_deref()).is_some() + { + description = loader_description + .clone() + .filter(|text| !text.is_empty()) + .or(description); + } + let mut help: String = yaml_help("source", name, &posix(&path)); + if let Some((loader_name, (_, loader_path, false))) = loader { + help = join_helps(&[ + help, + format!( + "or describe its loader '{loader_name}' in {} with a docstring or \ + @loader(description=\"What one row of {name} represents\")", + posix(loader_path) + ), + ]); + } + diagnostics.described( + ("source", name, Some("source")), + description.as_deref(), + || diagnostics.yaml_location(&path, contents, name), + &help, + ) +} + +/// A SQL or Python function's P010, located at its header or `@udf` decorator. +fn function_description<'py>( + diagnostics: &Diagnostics<'_, 'py>, + function: &crate::bindings::_helpers::compiled_project::function_views::FunctionRecord, +) -> PyResult>> { + let py: Python<'py> = diagnostics.py; + let attached = &function.attached; + let name: &str = &attached.name; + let function_file: &Bound<'py, PyAny> = attached.function_file.bind(py); + let path: String = function_file.getattr("relative_path")?.str()?.extract()?; + let contents: String = function_file.getattr("contents")?.extract()?; + let resource_type: &str = if attached.return_columns.is_empty() { + "udf" + } else { + "table_fn" + }; + if attached.python.is_some() { + let help: String = snippet_help( + &format!("to describe function '{name}'"), + &format!( + "give its function in {} a docstring or add this to @udf", + posix(&path) + ), + &[format!("@udf(description=\"What {name} returns\", ...)")], + ); + return diagnostics.described( + ("function", name, Some(resource_type)), + attached.description.as_deref(), + || { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("contents", &contents)?; + arguments.set_item("decorator", PYTHON_UDF_DECORATOR)?; + let (line, column): (usize, usize) = diagnostics + .settings + .locators + .bind(py) + .call_method("decorator_location", (), Some(&arguments))? + .extract()?; + diagnostics.location(&path, (line, column), None) + }, + &help, + ); + } + let entry: String = format!("description \"What {name} returns\""); + let help: String = header_help(&HeaderHelp { + kind: "function", + name, + statement: "FUNCTION", + entry: &entry, + path: &posix(&path), + description_only: false, + }); + diagnostics.described( + ("function", name, Some(resource_type)), + attached.description.as_deref(), + || diagnostics.header_location(&path, &contents), + &help, + ) +} + +fn config_text(values: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match values.get_item(key)? { + Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} + +fn cursor_without_inputs<'py>( + project: &NativeCompiledProject, + diagnostics: &Diagnostics<'_, 'py>, +) -> PyResult>> { + let py: Python<'py> = diagnostics.py; + let mut errors: Vec> = Vec::new(); + for record in project.inputs() { + let values: Bound<'py, PyDict> = + record.config.bind(py).getattr("values")?.downcast_into()?; + let kinds: Vec<&str> = record + .references + .iter() + .map(|(kind, _, _, _)| kind.as_str()) + .collect(); + let materialized: Option = config_text(&values, "materialized")?; + let cursor: Option = config_text(&values, CURSOR_KEY)?; + let strategy: Option = config_text(&values, "microbatch_strategy")?; + let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); + let name: String = model_file + .getattr("file_path")? + .getattr("stem")? + .extract()?; + let Some((message, help)) = cursor_lacks_inputs( + &name, + &CursorModel { + materialized: materialized.as_deref(), + cursor: cursor.as_deref(), + microbatch_strategy: strategy.as_deref(), + has_cursor_inputs: values + .get_item(CURSOR_INPUTS_KEY)? + .is_some_and(|value| !value.is_none()), + reference_kinds: &kinds, + }, + ) else { + continue; + }; + let path: String = model_file.getattr("relative_path")?.str()?.extract()?; + let contents: String = model_file.getattr("contents")?.extract()?; + let span: Option<(usize, usize)> = + model_header_entry_span(&contents, CURSOR_KEY).map_err(PyValueError::new_err)?; + let start: TextPosition = text_position(&contents, span.map_or(0, |span| span.0)); + let location: Bound<'py, PyAny> = diagnostics.location( + &path, + (start.line, start.column), + span.map(|(begin, end)| (start.line, start.column + end - begin)), + )?; + errors.push(diagnostics.diagnostic( + (CURSOR_WITHOUT_INPUTS_CODE, "error"), + (&message, Some(help)), + (Some("model"), &name), + location, + )?); + } + Ok(errors) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs new file mode 100644 index 0000000000..1b3587b063 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs @@ -0,0 +1,159 @@ +//! `build_scenario_inputs`: SQL scenarios expanded, classified and checked natively. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::compiler::main::sql_scenario_classification::extract_scenario; +use sqlbuild_analysis::compiler::models::{SqlScenarioCtes, SqlTestCteText}; +use sqlbuild_attachments::test_targets::main::scenario_source_violation::scenario_source_violation; +use sqlbuild_attachments::test_targets::models::ScenarioCteSources; +use sqlbuild_sqltext::sql_references::models::SqlReference; + +use crate::bindings::_helpers::compiled_project::model_loop::compile_error; +use crate::bindings::_helpers::compiled_project::model_loop_calls::origin_references; +use crate::bindings::_helpers::compiled_project::source_loop::{ + AuthoredExpansion, expanded_authored_sql, scoped_declarations, +}; +use crate::bindings::_helpers::compiled_project::sql_test_loop::{ + FileOrigin, TestScope, assertion_targets, checked_body, source_names, unique, +}; + +/// A scenario file's paths, discovered file, contents, SQL body and name. +pub(crate) type ScenarioRow = (String, String, Py, String, String, String); + +impl<'py> TestScope<'_, 'py> { + /// Every scenario expanded, classified and checked, in discovery order. + pub(crate) fn attached_scenarios(&self, rows: &[ScenarioRow]) -> PyResult>> { + rows.iter().map(|row| self.scenario(row)).collect() + } + + fn scenario(&self, row: &ScenarioRow) -> PyResult> { + let py: Python<'py> = self.context.host.py(); + let (file_path, relative_path, scenario_file, contents, sql_body, name) = row; + let resource: Bound<'py, PyAny> = self + .context + .classes + .visibility + .resource_identity + .call1((self.settings.scenario_kind.bind(py), name))?; + let declarations: Bound<'py, PyAny> = + scoped_declarations(self.context, file_path, &resource)?; + let expansion: AuthoredExpansion<'py> = expanded_authored_sql( + self.model_loop, + self.context, + (sql_body, file_path), + &declarations, + )?; + let stem: &str = std::path::Path::new(file_path) + .file_stem() + .and_then(|stem| stem.to_str()) + .unwrap_or_default(); + let sql: String = checked_body( + self.model_loop, + self.context, + &expansion.sql, + &format!("SQL scenario '{stem}'"), + )?; + let ctes: SqlScenarioCtes = extract_scenario(&sql, relative_path, &self.model_loop.syntax) + .map_err(|message| compile_error(self.context.classes, &message, None))?; + let origin: FileOrigin<'_> = (file_path, relative_path, contents); + self.checked_scenario_sources(&ctes, origin)?; + let assertions: Vec = texts(&ctes.assertions); + let targets: Vec = + assertion_targets(self.context, self.model_loop, &assertions, origin)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("scenario_file", scenario_file.bind(py))?; + fields.set_item("sql_body", &sql)?; + fields.set_item("authored_ctes", self.scenario_ctes(&ctes.authored)?)?; + fields.set_item("expected_ctes", self.scenario_ctes(&ctes.expected)?)?; + fields.set_item("assertion_ctes", self.scenario_ctes(&ctes.assertions)?)?; + fields.set_item( + "source_fixture_names", + PyTuple::new(py, &ctes.source_fixtures)?, + )?; + fields.set_item("ref_fixture_names", PyTuple::new(py, &ctes.ref_fixtures)?)?; + fields.set_item("seed_fixture_names", PyTuple::new(py, &ctes.seed_fixtures)?)?; + fields.set_item( + "dbt_ref_fixture_names", + PyTuple::new(py, &ctes.dbt_ref_fixtures)?, + )?; + fields.set_item( + "expected_model_names", + PyTuple::new(py, &ctes.expected_models)?, + )?; + fields.set_item("assertion_names", PyTuple::new(py, &ctes.assertion_names)?)?; + fields.set_item("assertion_target_model_names", PyTuple::new(py, &targets)?)?; + fields.set_item( + "target_model_names", + PyTuple::new( + py, + unique(ctes.expected_models.iter().cloned().chain(targets)), + )?, + )?; + fields.set_item("declaration_usages", &expansion.usages)?; + Ok(self + .settings + .scenario_input + .bind(py) + .call((), Some(&fields))? + .unbind()) + } + + /// `_validate_scenario_source_references`: disallowed source reads, then a scan's error. + fn checked_scenario_sources( + &self, + ctes: &SqlScenarioCtes, + origin: FileOrigin<'_>, + ) -> PyResult<()> { + let checks: usize = ctes.expected.len() + ctes.assertions.len(); + let mut sources: Vec = Vec::new(); + let mut failure: Option = None; + let all = ctes + .expected + .iter() + .chain(&ctes.assertions) + .chain(&ctes.authored); + for (position, (name, body)) in all.enumerate() { + let references: Vec = + match origin_references(self.context, &self.model_loop.syntax, body, origin) { + Ok(references) => references, + Err(error) => { + failure = Some(error); + break; + } + }; + sources.push(ScenarioCteSources { + name: name.clone(), + check: position < checks, + sources: source_names(&references), + }); + } + if let Some(message) = scenario_source_violation(origin.1, &sources, &self.catalog.sources) + { + return Err(compile_error(self.context.classes, &message, None)); + } + failure.map_or(Ok(()), Err) + } + + fn scenario_ctes(&self, ctes: &[(String, String)]) -> PyResult> { + let py: Python<'py> = self.context.host.py(); + let mut records: Vec> = Vec::with_capacity(ctes.len()); + for (name, body) in ctes { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("name", name)?; + fields.set_item("sql_body", body)?; + records.push( + self.settings + .scenario_cte + .bind(py) + .call((), Some(&fields))?, + ); + } + PyTuple::new(py, records) + } +} + +fn texts(ctes: &[(String, String)]) -> Vec { + ctes.iter() + .map(|(name, body)| SqlTestCteText(name.clone(), body.clone())) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs new file mode 100644 index 0000000000..5c786f48e3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs @@ -0,0 +1,285 @@ +//! Seeds paired with their declarations, held by the compiled project, and their views. + +use pyo3::exceptions::{PyIndexError, PyRuntimeError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::PyTuple; +use pyo3::{pyclass, pymethods}; +use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; + +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + DestinationFact, defaulted_destination, object_key, relation_location, +}; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + InputViewClasses, view_classes, +}; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; + +const SEED_RESOURCE: &str = "seed"; +const NAME_ATTRIBUTE: &str = "name"; +const FILE_PATH_ATTRIBUTE: &str = "file_path"; +const STEM_ATTRIBUTE: &str = "stem"; +const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; + +/// `(database, schema, logical database, logical schema)` of a seed's destination. +pub(crate) type SeedNamespace = ( + Option, + Option, + Option, + Option, +); + +/// One seed CSV paired with its declaration; the discovered objects are discovery's. +pub(crate) struct SeedRecord { + /// The discovered CSV file. + seed_file: Py, + /// The discovered declaration, whose spec carries user-authored column metadata. + schema_entry: Py, + /// The schema file declaring the seed. + schema_file: Py, + name: String, + /// The destination native project assembly resolved, once it ran. + destination: Option, + key: PyOnceLock>, +} + +/// Pair each declaration with the CSV file of its name, raising for the first without one. +pub(crate) fn paired_seeds( + declarations: Vec<(Py, Py)>, + seed_files: &[Py], + compile_error: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = compile_error.py(); + let mut names: Vec = Vec::with_capacity(declarations.len()); + for (entry, _) in &declarations { + names.push(entry.bind(py).getattr(NAME_ATTRIBUTE)?.extract()?); + } + let mut stems: Vec = Vec::with_capacity(seed_files.len()); + for seed_file in seed_files { + stems.push( + seed_file + .bind(py) + .getattr(FILE_PATH_ATTRIBUTE)? + .getattr(STEM_ATTRIBUTE)? + .extract()?, + ); + } + let pairs: Vec = match pair_seed_declarations(&names, &stems) { + Ok(pairs) => pairs, + Err(missing) => { + let schema_file: &Py = &declarations[missing].1; + let relative_path: String = schema_file + .bind(py) + .getattr(RELATIVE_PATH_ATTRIBUTE)? + .str()? + .extract()?; + let message: String = format!( + "Seed declaration '{}' in {relative_path} has no matching CSV file under seeds/", + names[missing] + ); + return Err(PyErr::from_value(compile_error.call1((message,))?)); + } + }; + let mut seeds: Vec = Vec::with_capacity(pairs.len()); + for (((schema_entry, schema_file), name), file_index) in + declarations.into_iter().zip(names).zip(pairs) + { + seeds.push(SeedRecord { + seed_file: seed_files[file_index].clone_ref(py), + schema_entry, + schema_file, + name, + destination: None, + key: PyOnceLock::new(), + }); + } + Ok(seeds) +} + +impl SeedRecord { + /// The seed's name, CSV file, declaration and declaring schema file. + pub(crate) fn parts(&self) -> (&str, &Py, &Py, &Py) { + ( + &self.name, + &self.seed_file, + &self.schema_entry, + &self.schema_file, + ) + } + + /// Keep the destination native project assembly resolved for the seed. + pub(crate) fn set_namespace(&mut self, namespace: SeedNamespace) { + let (database, schema, logical_database, logical_schema) = namespace; + self.destination = Some(DestinationFact { + database, + schema, + name: self.name.clone(), + qualified_name: None, + logical_schema, + logical_database, + }); + } +} + +fn read_seed<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&SeedRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + let seed: &SeedRecord = owner + .seeds() + .get(index) + .ok_or_else(|| PyIndexError::new_err(index))?; + read(seed, py) +} + +/// One attached seed, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeSeedInput { + owner: Py, + index: usize, +} + +impl NativeSeedInput { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +#[pymethods] +impl NativeSeedInput { + #[getter] + fn seed_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.seed_file.clone_ref(py)) + }) + } + + #[getter] + fn schema_entry(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.schema_entry.clone_ref(py)) + }) + } + + #[getter] + fn schema_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.schema_file.clone_ref(py)) + }) + } +} + +/// One assembled seed, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeCompiledSeed { + owner: Py, + index: usize, + /// The destination with target defaults applied, replacing the assembled one. + defaulted: Option, + destination: PyOnceLock>, +} + +impl NativeCompiledSeed { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self { + owner, + index, + defaulted: None, + destination: PyOnceLock::new(), + } + } + + fn current_destination(&self, py: Python<'_>) -> PyResult { + if let Some(defaulted) = &self.defaulted { + return Ok(defaulted.clone()); + } + read_seed(&self.owner, self.index, py, |seed, _| { + seed.destination + .clone() + .ok_or_else(|| PyRuntimeError::new_err("the seed was not assembled")) + }) + } +} + +#[pymethods] +impl NativeCompiledSeed { + /// The seed's planner-facing `CompiledObjectKey`. + #[getter] + fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_seed(&self.owner, self.index, py, |seed, py| { + seed.key + .get_or_try_init(py, || { + object_key(&classes.compiled, SEED_RESOURCE, &seed.name) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { + PyTuple::empty(py) + } + + #[getter] + fn name(&self, py: Python<'_>) -> PyResult { + read_seed(&self.owner, self.index, py, |seed, _| Ok(seed.name.clone())) + } + + #[getter] + fn seed_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.seed_file.clone_ref(py)) + }) + } + + #[getter] + fn schema_entry(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.schema_entry.clone_ref(py)) + }) + } + + #[getter] + fn schema_file(&self, py: Python<'_>) -> PyResult> { + read_seed(&self.owner, self.index, py, |seed, py| { + Ok(seed.schema_file.clone_ref(py)) + }) + } + + /// The seed's planner-facing `CompiledRelationLocation`. + #[getter] + fn destination(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + self.destination + .get_or_try_init(py, || { + relation_location(&classes.compiled, &self.current_destination(py)?) + }) + .map(|destination| destination.clone_ref(py)) + } + + /// This seed with the adapter's default schema and database filled into its destination. + fn with_target_defaults( + &self, + py: Python<'_>, + defaults: (Option, Option), + render: Bound<'_, PyAny>, + ) -> PyResult { + let defaulted: DestinationFact = + defaulted_destination(&self.current_destination(py)?, defaults, &render)?; + Ok(Self { + owner: self.owner.clone_ref(py), + index: self.index, + defaulted: Some(defaulted), + destination: PyOnceLock::new(), + }) + } + + #[getter(external)] + fn is_external(&self) -> bool { + false + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs new file mode 100644 index 0000000000..ade81a99f0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs @@ -0,0 +1,489 @@ +//! `build_source_inputs`: every discovered source attached natively into the compiled project. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; +use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::templates::models::TemplateOptions; + +use crate::bindings::_helpers::compiled_project::model_loop::{ + LoopContext, NativeModelLoop, compile_error, record_reads, +}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; +use crate::bindings::_helpers::compiled_project::model_loop_macros::{ + argument_references as macro_argument_references, expanded_with, +}; +use crate::bindings::_helpers::compiled_project::source_views::{AttachedSource, replaced}; +use crate::bindings::_helpers::model_config::config_errors::native_config_error; +use crate::bindings::_helpers::model_config::config_templates::{ + PythonHost, Stop, TemplateSources, expanded, template_error, +}; +use crate::bindings::_helpers::project_assembly::resources::syntax_error_message; +use crate::bindings::_helpers::render::declaration_expansion::{ + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, + expand_declarations, +}; +use crate::bindings::_helpers::scopes::declaration_contexts::NativeDeclarationContexts; +use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; +use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolated}; + +const MACRO_TOKEN: char = '@'; +const FILE_PATH_ARGUMENT: &str = "file_path"; +const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; +const NATIVE_CONTEXTS_ATTRIBUTE: &str = "native_contexts"; +const GENERIC_DIALECT: &str = "generic"; +const EXPRESSION_STATEMENTS: [&str; 3] = ["select", "with", "values"]; + +/// One discovered source: its file's path, the discovered file and the entry. +pub(crate) type SourceRow = (String, Py, Py); + +/// `(default contract, whether expressions are syntax-checked, source kind)`. +pub(crate) type SourceSettings = (Option, bool, Py); + +/// Attach every source in order, as `build_source_inputs` does. +pub(crate) fn attached_sources<'py>( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, 'py>, + rows: Vec, + settings: SourceSettings, +) -> PyResult> { + let (default_contract, validate, source_kind) = settings; + let py: Python<'py> = context.host.py(); + let source_kind: Bound<'py, PyAny> = source_kind.into_bound(py); + let mut attached: Vec = Vec::with_capacity(rows.len()); + for (file_path, source_file, entry) in rows { + let entry: Bound<'py, PyAny> = entry.into_bound(py); + let name: String = entry.getattr("name")?.extract()?; + let consumer: Bound<'py, PyAny> = context + .classes + .visibility + .resource_identity + .call1((&source_kind, &name))?; + let declarations: Bound<'py, PyAny> = scoped_declarations(context, &file_path, &consumer)?; + let (expression, usages) = match entry.getattr("expression")?.extract::>()? { + Some(sql) => { + let expansion: AuthoredExpansion<'py> = + expanded_authored_sql(model_loop, context, (&sql, &file_path), &declarations)?; + (Some(expansion.sql), expansion.usages) + } + None => (None, PyTuple::empty(py)), + }; + let label = |field: &str| format!("source {name} {field}"); + let database: Option = + template_text(context, &entry, "database", &label("database"))?; + let schema: Option = template_text(context, &entry, "schema", &label("schema"))?; + let table: Option = template_text(context, &entry, "table", &label("table"))?; + let description: Option = + template_text(context, &entry, "description", &label("description"))?; + let meta: Bound<'py, PyAny> = + template_value(context, &entry.getattr("meta")?, &label("meta"))?; + let mut columns: Vec> = Vec::new(); + for column in entry.getattr("columns")?.try_iter()? { + columns.push(expanded_column(context, &name, &column?)?); + } + let mut audits: Vec> = Vec::new(); + for audit in entry.getattr("audits")?.try_iter()? { + let audit: Bound<'py, PyAny> = audit?; + let definition: String = audit.getattr("definition_name")?.extract()?; + audits.push(expanded_audit( + context, + &audit, + &format!("source {name} audit {definition}"), + )?); + } + if let Some(sql) = &expression { + let label: String = format!("Source expression '{name}'"); + if let IntrinsicCheck::Rejected(message) = + intrinsic_free(model_loop.python, sql, &model_loop.reserved_markers, &label) + { + return Err(compile_error(context.classes, &message, None)); + } + if validate { + checked_expression(context, (&name, &file_path, sql))?; + } + } + let contract: Option = entry + .getattr("contract")? + .extract::>()? + .or_else(|| default_contract.clone()); + attached.push(AttachedSource { + source_file, + entry: entry.unbind(), + name, + contract, + database, + schema, + table, + expression, + description, + meta: meta.unbind(), + columns: PyTuple::new(py, columns)?.unbind(), + audits: PyTuple::new(py, audits)?.unbind(), + usages: usages.unbind(), + }); + } + Ok(attached) +} + +/// A source's declaration context, or the public declarations when none are indexed. +pub(crate) fn scoped_declarations<'py>( + context: &LoopContext<'_, 'py>, + file_path: &str, + consumer: &Bound<'py, PyAny>, +) -> PyResult> { + let visibility = &context.classes.visibility; + if visibility.projects { + let contexts: Bound<'py, NativeDeclarationContexts> = visibility + .resolver + .getattr(NATIVE_CONTEXTS_ATTRIBUTE)? + .downcast_into()?; + let (resolved, _) = + contexts + .get() + .resolved_context(&visibility.resolver, file_path, Some(consumer))?; + return Ok(resolved); + } + let fields: Bound<'py, PyDict> = PyDict::new(consumer.py()); + fields.set_item("enums", &visibility.public_enums)?; + fields.set_item("constants", &visibility.public_constants)?; + context + .classes + .macros + .declaration_context + .call((), Some(&fields)) +} + +/// Authored SQL after expansion, with its usages and typed macro argument references. +pub(crate) struct AuthoredExpansion<'py> { + pub(crate) sql: String, + pub(crate) usages: Bound<'py, PyTuple>, + pub(crate) argument_references: Vec, +} + +/// `declaration_file_expansion`: a declaration file's folder context, consumed by `identity`. +pub(crate) fn file_declarations<'py>( + context: &LoopContext<'_, 'py>, + file_path: &str, + identity: &Bound<'py, PyAny>, +) -> PyResult> { + let visibility = &context.classes.visibility; + let py: Python<'py> = identity.py(); + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("consumer", identity)?; + if visibility.projects { + let contexts: Bound<'py, NativeDeclarationContexts> = visibility + .resolver + .getattr(NATIVE_CONTEXTS_ATTRIBUTE)? + .downcast_into()?; + let (resolved, _) = + contexts + .get() + .resolved_context(&visibility.resolver, file_path, None)?; + return Ok(replaced(py, &resolved, &fields)?.into_bound(py)); + } + fields.set_item("enums", &visibility.public_enums)?; + fields.set_item("constants", &visibility.public_constants)?; + context + .classes + .macros + .declaration_context + .call((), Some(&fields)) +} + +/// `expand_authored_sql_result`: interpolate, splice declarations and expand macros. +pub(crate) fn expanded_authored_sql<'py>( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, 'py>, + authored: (&str, &str), + declarations: &Bound<'py, PyAny>, +) -> PyResult> { + let (sql, file_path) = authored; + let py: Python<'py> = declarations.py(); + let hooks = &context.classes.sql_hooks; + let (interpolated_sql, reads, error) = interpolated( + model_loop.python, + InterpolationSources( + hooks.effective_vars.clone(), + hooks.environment.clone(), + None, + hooks.render_project_var.clone(), + ), + sql, + file_path, + )?; + let reads: Vec<(String, String)> = reads + .into_iter() + .map(|(kind, name)| (kind.to_owned(), name)) + .collect(); + record_reads(context, &reads)?; + if let Some(message) = error { + return Err(compile_error(context.classes, &message, None)); + } + let interpolated_sql: String = interpolated_sql.unwrap_or_else(|| sql.to_owned()); + let consumer: Bound<'py, PyAny> = declarations.getattr("consumer")?; + let (references, stop) = declaration_scan(model_loop.python, &interpolated_sql); + let mut usages: Vec> = Vec::new(); + let mut argument_references: Vec = Vec::new(); + let mut expanded_sql: String = interpolated_sql.clone(); + if !references.is_empty() || stop.is_some() { + let file_keyword: Bound<'py, PyDict> = PyDict::new(py); + file_keyword.set_item( + FILE_PATH_ARGUMENT, + context.classes.path_class.call1((file_path,))?, + )?; + let render_constant: Bound<'py, PyAny> = context.classes.partial.call( + (context.host.getattr(CONSTANT_TEXT_CALLBACK)?,), + Some(&file_keyword), + )?; + let classes: ExpansionClasses<'py> = ExpansionClasses { + usage_record: context.classes.usage_record.clone(), + compile_error: context.classes.compile_error.clone(), + }; + let expanded: ExpandedDeclarations<'py> = expand_declarations(&DeclarationExpansion { + sql: &interpolated_sql, + references: &references, + stop, + context: declarations, + consumer: ExpansionConsumer { + identity: (!consumer.is_none()).then(|| consumer.clone()), + relationship_visibility: false, + path: file_path, + }, + render_constant: &render_constant, + classes: &classes, + })?; + expanded_sql = expanded.spliced.sql; + usages.extend(expanded.usages); + } + if expanded_sql.contains(MACRO_TOKEN) { + let resolver: &Bound<'py, PyAny> = &context.classes.visibility.resolver; + let macro_declarations: Bound<'py, PyAny> = if resolver.is_none() { + py.None().into_bound(py) + } else { + let contexts: Bound<'py, NativeDeclarationContexts> = resolver + .getattr(NATIVE_CONTEXTS_ATTRIBUTE)? + .downcast_into()?; + contexts + .get() + .resolved_context(resolver, file_path, None)? + .0 + }; + let output = expanded_with( + context, + (¯o_declarations, &consumer), + file_path, + &expanded_sql, + )?; + argument_references = macro_argument_references(context, &output)?; + usages.extend(output.usages.into_iter().map(|usage| usage.into_bound(py))); + expanded_sql = output.sql; + } + let unique: Bound<'py, PyDict> = PyDict::new(py); + for usage in usages { + unique.set_item(usage, py.None())?; + } + Ok(AuthoredExpansion { + sql: expanded_sql, + usages: PyTuple::new(py, unique.keys())?, + argument_references, + }) +} + +/// `_expand_source_template_value`: a text field with its `${...}` templates expanded. +fn template_text( + context: &LoopContext<'_, '_>, + owner: &Bound<'_, PyAny>, + field: &str, + label: &str, +) -> PyResult> { + let raw: Bound<'_, PyAny> = owner.getattr(field)?; + if raw.is_none() { + return Ok(None); + } + Ok(Some( + template_value(context, &raw, label)?.str()?.extract()?, + )) +} + +/// `_expand_source_template_object`: `${...}` templates expanded without context values. +fn template_value<'py>( + context: &LoopContext<'_, 'py>, + value: &Bound<'py, PyAny>, + label: &str, +) -> PyResult> { + expanded_template(context, value, label, false) +} + +/// `expand_config_templates` without context values; `preserve` keeps `@@CTX` tokens. +pub(crate) fn expanded_template<'py>( + context: &LoopContext<'_, 'py>, + value: &Bound<'py, PyAny>, + label: &str, + preserve: bool, +) -> PyResult> { + let py: Python<'py> = value.py(); + let hooks = &context.classes.sql_hooks; + let host: PythonHost<'py> = PythonHost::new( + py, + TemplateSources( + hooks.effective_vars.clone(), + hooks.environment.clone(), + PyDict::new(py), + ), + label, + ); + let options: TemplateOptions = TemplateOptions { + allow_context: false, + preserve_context_tokens: preserve, + preserve_unknown_context: false, + }; + let result: Result, Stop> = expanded(&host, value, options); + let reads: Vec<(String, String)> = host + .into_reads() + .into_iter() + .map(|(kind, name)| (kind.to_owned(), name)) + .collect(); + record_reads(context, &reads)?; + match result { + Ok(value) => Ok(value), + Err(Stop::Python(error)) => Err(error), + Err(Stop::Failure(error)) => { + let error: Bound<'py, PyAny> = native_config_error(py, template_error(&error, label))? + .into_bound(py) + .into_any(); + Err(raised_config_error(context, &error)) + } + } +} + +fn raised_config_error(context: &LoopContext<'_, '_>, error: &Bound<'_, PyAny>) -> PyErr { + let raised = (|| { + let keywords: Bound<'_, PyDict> = PyDict::new(error.py()); + keywords.set_item("error", error)?; + context.classes.config_error.call((), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +/// `expand_source_column_templates`. +fn expanded_column( + context: &LoopContext<'_, '_>, + source: &str, + column: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = column.py(); + let column_name: String = column.getattr("name")?.str()?.extract()?; + let label = |field: &str| format!("source {source} column {column_name} {field}"); + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item( + "type", + template_text(context, column, "type", &label("type"))?, + )?; + changes.set_item( + "description", + template_text(context, column, "description", &label("description"))?, + )?; + changes.set_item( + "meta", + template_value(context, &column.getattr("meta")?, &label("meta"))?, + )?; + let mut audits: Vec> = Vec::new(); + for audit in column.getattr("audits")?.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let definition: String = audit.getattr("definition_name")?.extract()?; + audits.push(expanded_audit( + context, + &audit, + &label(&format!("audit {definition}")), + )?); + } + changes.set_item("audits", PyTuple::new(py, audits)?)?; + replaced_spec(column, &changes) +} + +/// `expand_schema_audit_instance_templates`. +fn expanded_audit( + context: &LoopContext<'_, '_>, + audit: &Bound<'_, PyAny>, + label: &str, +) -> PyResult> { + let changes: Bound<'_, PyDict> = PyDict::new(audit.py()); + changes.set_item( + "arguments", + template_value( + context, + &audit.getattr("arguments")?, + &format!("{label} arguments"), + )?, + )?; + changes.set_item( + "description", + template_text( + context, + audit, + "description", + &format!("{label} description"), + )?, + )?; + replaced_spec(audit, &changes) +} + +fn replaced_spec(value: &Bound<'_, PyAny>, changes: &Bound<'_, PyDict>) -> PyResult> { + replaced(value.py(), value, changes) +} + +/// `validate_source_expression_syntax`: the expression parses as a FROM target. +fn checked_expression(context: &LoopContext<'_, '_>, source: (&str, &str, &str)) -> PyResult<()> { + let (name, file_path, expression) = source; + parse_checked( + context, + &format!("SELECT * FROM {}", expression_relation(expression)), + &format!("SQL syntax error in source expression '{name}' ({file_path})"), + ) +} + +/// `_validate_sql_syntax_with_message` for SQL a non-model resource owns. +pub(crate) fn parse_checked( + context: &LoopContext<'_, '_>, + sql: &str, + error_prefix: &str, +) -> PyResult<()> { + let py: Python<'_> = context.host.py(); + let check: SyntaxCheck = SyntaxCheck { + sql: sql.to_owned(), + placeholders: Vec::new(), + }; + let Some(message) = syntax_error_message(py, &check, GENERIC_DIALECT, SyntaxMode::Parse)? + else { + return Ok(()); + }; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", &context.classes.source_syntax_help)?; + let error: Bound<'_, PyAny> = context + .classes + .compile_error + .call((format!("{error_prefix}: {message}"),), Some(&keywords))?; + Err(PyErr::from_value(error)) +} + +/// `_render_source_expression_relation`: the expression as a table factor. +fn expression_relation(expression: &str) -> String { + let trimmed: &str = python_strip(expression); + let stripped: &str = python_strip(trimmed.strip_suffix(';').unwrap_or(trimmed)); + if stripped.starts_with('(') { + return stripped.to_owned(); + } + let lowered: String = stripped.to_lowercase(); + if EXPRESSION_STATEMENTS + .iter() + .any(|statement| lowered.starts_with(statement)) + { + return format!("({stripped})"); + } + stripped.to_owned() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs new file mode 100644 index 0000000000..83dfeff912 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -0,0 +1,265 @@ +//! Sources attached by the native loop, held by the compiled project, and their views. + +use pyo3::exceptions::PyIndexError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyclass, pymethods}; + +use crate::bindings::_helpers::compiled_project::compiled_model_view::object_key; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + InputViewClasses, view_classes, +}; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; + +const SOURCE_RESOURCE: &str = "source"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const REPLACE_FUNCTION: &str = "replace"; + +/// `(database, schema)` native project assembly resolved for a source, or `None`. +pub(crate) type SourceNamespace = Option<(Option, Option)>; + +/// What the source loop attached for one source: its expanded metadata and usages. +pub(crate) struct AttachedSource { + /// The discovered source file. + pub(crate) source_file: Py, + /// The discovered entry; its spec carries user-authored metadata. + pub(crate) entry: Py, + pub(crate) name: String, + pub(crate) contract: Option, + pub(crate) database: Option, + pub(crate) schema: Option, + pub(crate) table: Option, + pub(crate) expression: Option, + pub(crate) description: Option, + /// Template-expanded user metadata. + pub(crate) meta: Py, + /// Template-expanded column specs, which carry user metadata and audit arguments. + pub(crate) columns: Py, + /// Template-expanded table audit specs, which carry user audit arguments. + pub(crate) audits: Py, + /// Usage records, the identities the scope index is built from. + pub(crate) usages: Py, +} + +/// One attached source held by the compiled project. +pub(crate) struct SourceRecord { + attached: AttachedSource, + /// The namespace native project assembly resolved, once it ran. + namespace: Option, + attached_entry: PyOnceLock>, + assembled_entry: PyOnceLock>, + key: PyOnceLock>, +} + +impl SourceRecord { + pub(crate) fn new(attached: AttachedSource) -> Self { + Self { + attached, + namespace: None, + attached_entry: PyOnceLock::new(), + assembled_entry: PyOnceLock::new(), + key: PyOnceLock::new(), + } + } + + /// The attached source. + pub(crate) fn attached(&self) -> &AttachedSource { + &self.attached + } + + /// Keep the namespace native project assembly resolved for the source. + pub(crate) fn set_namespace(&mut self, namespace: SourceNamespace) { + self.namespace = Some(namespace); + } + + /// The attached `SourceEntry`: the discovered entry with its expanded metadata. + fn attached_entry(&self, py: Python<'_>) -> PyResult> { + self.attached_entry + .get_or_try_init(py, || { + let source: &AttachedSource = &self.attached; + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item("contract", &source.contract)?; + changes.set_item("database", &source.database)?; + changes.set_item("schema", &source.schema)?; + changes.set_item("table", &source.table)?; + changes.set_item("expression", &source.expression)?; + changes.set_item("description", &source.description)?; + changes.set_item("meta", source.meta.bind(py))?; + changes.set_item("columns", source.columns.bind(py))?; + changes.set_item("audits", source.audits.bind(py))?; + replaced(py, source.entry.bind(py), &changes) + }) + .map(|entry| entry.clone_ref(py)) + } + + /// The assembled `SourceEntry`: the attached entry in its resolved namespace. + fn assembled_entry(&self, py: Python<'_>) -> PyResult> { + let attached: Py = self.attached_entry(py)?; + let Some(Some((database, schema))) = &self.namespace else { + return Ok(attached); + }; + self.assembled_entry + .get_or_try_init(py, || { + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item("database", database)?; + changes.set_item("schema", schema)?; + replaced(py, attached.bind(py), &changes) + }) + .map(|entry| entry.clone_ref(py)) + } +} + +/// `dataclasses.replace(value, **changes)`. +pub(crate) fn replaced( + py: Python<'_>, + value: &Bound<'_, PyAny>, + changes: &Bound<'_, PyDict>, +) -> PyResult> { + Ok(py + .import(DATACLASSES_MODULE)? + .getattr(REPLACE_FUNCTION)? + .call((value,), Some(changes))? + .unbind()) +} + +fn read_source<'py, T>( + owner: &Py, + index: usize, + py: Python<'py>, + read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + let source: &SourceRecord = owner + .sources() + .get(index) + .ok_or_else(|| PyIndexError::new_err(index))?; + read(source, py) +} + +/// One attached source, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeSourceInput { + owner: Py, + index: usize, +} + +impl NativeSourceInput { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +#[pymethods] +impl NativeSourceInput { + #[getter] + fn source_entry(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + source.attached_entry(py) + }) + } + + #[getter] + fn source_file(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.source_file.clone_ref(py)) + }) + } + + #[getter] + fn declaration_usages(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.usages.clone_ref(py)) + }) + } + + /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. + #[getter] + fn assembly_row( + &self, + py: Python<'_>, + ) -> PyResult<(String, bool, Option, Option)> { + read_source(&self.owner, self.index, py, |source, py| { + let attached: &AttachedSource = &source.attached; + Ok(( + attached.name.clone(), + !attached.entry.bind(py).getattr("loader")?.is_none(), + attached.database.clone(), + attached.schema.clone(), + )) + }) + } +} + +/// One assembled source, read from the compiled project that owns it. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeCompiledSource { + owner: Py, + index: usize, +} + +impl NativeCompiledSource { + pub(crate) fn new(owner: Py, index: usize) -> Self { + Self { owner, index } + } +} + +#[pymethods] +impl NativeCompiledSource { + /// The source's planner-facing `CompiledObjectKey`. + #[getter] + fn key(&self, py: Python<'_>) -> PyResult> { + let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; + read_source(&self.owner, self.index, py, |source, py| { + source + .key + .get_or_try_init(py, || { + object_key(&classes.compiled, SOURCE_RESOURCE, &source.attached.name) + }) + .map(|key| key.clone_ref(py)) + }) + } + + #[getter] + fn deps<'py>(&self, py: Python<'py>) -> Bound<'py, PyTuple> { + PyTuple::empty(py) + } + + #[getter] + fn name(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, _| { + Ok(source.attached.name.clone()) + }) + } + + #[getter] + fn source_entry(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + source.assembled_entry(py) + }) + } + + #[getter] + fn source_file(&self, py: Python<'_>) -> PyResult> { + read_source(&self.owner, self.index, py, |source, py| { + Ok(source.attached.source_file.clone_ref(py)) + }) + } +} + +/// Keep `attached` in `owner`; return each source's view. +pub(crate) fn retain_sources( + owner: &Bound<'_, NativeCompiledProject>, + attached: Vec, +) -> Vec { + let mut project = owner.borrow_mut(); + let first: usize = project.sources().len(); + let count: usize = attached.len(); + for source in attached { + project.retain_source(SourceRecord::new(source)); + } + drop(project); + (first..first + count) + .map(|index| NativeSourceInput::new(owner.clone().unbind(), index)) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs new file mode 100644 index 0000000000..8fe1e98820 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs @@ -0,0 +1,1089 @@ +//! `build_test_inputs`: SQL test blocks expanded, classified and checked natively. + +use std::collections::{HashMap, HashSet}; +use std::convert::Infallible; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::compiled_project::main::text_position::text_position; +use sqlbuild_analysis::compiled_project::models::TextPosition; +use sqlbuild_analysis::compiler::main::sql_test_classification::extract_sql_tests; +use sqlbuild_analysis::compiler::models::{ + SqlTestClassification, SqlTestCteText, SqlTestExtractionFailure, SqlTestRequest, +}; +use sqlbuild_analysis::sql_test_attachment::main::authored_test_offset::authored_test_offset; +use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; +use sqlbuild_analysis::sql_test_attachment::main::sql_test_reads::sql_test_reads; +use sqlbuild_analysis::sql_test_attachment::main::stored_sql_tests::stored_sql_tests; +use sqlbuild_analysis::sql_test_attachment::main::test_without_target_model::test_without_target_model; +use sqlbuild_analysis::sql_test_attachment::main::unresolvable_test_references::unresolvable_test_references; +use sqlbuild_analysis::sql_test_attachment::models::{ + CteLocation, ModelTestCtes, PythonTextMatcher, SqlTestCheckFailure, SqlTestHelperProblem, + SqlTestReadFacts, +}; +use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; +use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; +use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; +use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; +use sqlbuild_attachments::functions::main::table_function_arity_error::table_function_arity_error; +use sqlbuild_attachments::test_parameters::main::parameter_references::parameter_references; +use sqlbuild_attachments::test_targets::main::unknown_test_target::unknown_test_target; +use sqlbuild_attachments::test_targets::models::{TargetCatalog, TestTargets}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names; +use sqlbuild_sqltext::compiler::models::DeclarationReferenceStop; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; + +use crate::bindings::_helpers::cache::sql_test_scan_store::SqlTestScanStore; +use crate::bindings::_helpers::compiled_project::model_loop::{ + LoopContext, NativeModelLoop, compile_error, helped_compile_error, +}; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + origin_references, report_invalid_call, +}; +use crate::bindings::_helpers::compiled_project::source_loop::{ + AuthoredExpansion, expanded_authored_sql, scoped_declarations, +}; +use crate::bindings::_helpers::compiled_project::source_views::replaced; + +const MODEL_MODE: &str = "model"; +const MACRO_MODE: &str = "macro"; +const TABLE_FUNCTION_MODE: &str = "table_fn"; +const REF_KIND: &str = "ref"; +const SOURCE_KIND: &str = "source"; +const UDF_KIND: &str = "udf"; +const TABLE_FUNCTION_KIND: &str = "table_fn"; +const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; +const OMITTED_CEREMONIAL_SELECT: &str = "\nSELECT 1"; +const HELPER_REFERENCE_CODE: &str = "P013"; +const SQL_TEST_RESOURCE: &str = "sql_test"; +const MAX_SQL_VALUE_SIZE: usize = 1_000_000; +const ENUM_VISIBILITY: &str = "enum_visibility"; +const CONSTANT_VISIBILITY: &str = "constant_visibility"; +const MACRO_RECORDS: &str = "macro_records"; +const MACRO_VISIBILITY: &str = "macro_visibility"; + +/// A test case's discovered case, name and parameter values in authored order. +pub(crate) type TestCaseRow = (Py, String, Vec<(String, Py)>); + +/// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. +pub(crate) type TestBlockRow = ( + Py, + Option, + String, + String, + Vec, + Vec, + Option<(usize, usize)>, +); + +/// A test file's paths, stem, discovered file, contents and blocks. +pub(crate) type TestFileRow = (String, String, String, Py, String, Vec); + +/// What every SQL test and scenario of one compile resolves and records with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct SqlTestSettings { + /// `CompileSqlTestInput`, its model and direct payloads, and `CompileSqlTestCte`. + pub(crate) test_input: Py, + pub(crate) model_payload: Py, + pub(crate) direct_payload: Py, + pub(crate) test_cte: Py, + /// `CompileSqlScenarioInput` and `CompileSqlScenarioCte`. + pub(crate) scenario_input: Py, + pub(crate) scenario_cte: Py, + /// `SqlTestMode` by value. + pub(crate) modes: Py, + /// The `ResourceKind`s of tests and scenarios, and the runtime `UsageKind`. + pub(crate) test_kind: Py, + pub(crate) scenario_kind: Py, + pub(crate) runtime_usage: Py, + /// The adapter's typed scalar renderer, returning `(text, None)` or `(None, error)`. + pub(crate) render_parameter: Py, + pub(crate) adapter_name: String, + /// `SourceLocation`, for helper problems located natively. + pub(crate) source_location: Py, + /// The shared scan store with the scan algorithm and lexical rules keying it, if any. + pub(crate) scan_store: Option>, + pub(crate) scan_key: (String, String), +} + +/// The loop, its context, the settings and the project names every test resolves against. +pub(crate) struct TestScope<'a, 'py> { + pub(crate) model_loop: &'a NativeModelLoop, + pub(crate) context: &'a LoopContext<'a, 'py>, + pub(crate) settings: &'a SqlTestSettings, + pub(crate) catalog: TargetCatalog, + pub(crate) functions: HashSet, + pub(crate) table_function_arities: HashMap, +} + +/// One test case expanded before extraction. +struct ExpandedTest<'a, 'py> { + file: &'a TestFileRow, + block: &'a TestBlockRow, + case: Option<&'a TestCaseRow>, + sql: String, + tested: Vec, + usages: Bound<'py, PyTuple>, +} + +/// The attached SQL test inputs, then scenario inputs. +pub(crate) type AttachedSqlTests = (Vec>, Vec>); + +/// `(file path, relative path, contents)` of an authored test or scenario file. +pub(crate) type FileOrigin<'a> = (&'a str, &'a str, &'a str); + +impl<'py> TestScope<'_, 'py> { + fn py(&self) -> Python<'py> { + self.context.host.py() + } + + fn error(&self, message: &str) -> PyErr { + compile_error(self.context.classes, message, None) + } + + /// Every test case expanded, the batch classified, then each test checked and recorded. + pub(crate) fn attached(&self, files: &[TestFileRow]) -> PyResult>> { + let mut expanded: Vec> = Vec::new(); + for file in files { + for block in &file.5 { + if block.5.is_empty() { + expanded.push(self.expanded(file, block, None)?); + } + for case in &block.5 { + expanded.push(self.expanded(file, block, Some(case))?); + } + } + } + let classified: Vec = self.classified(&expanded)?; + let mut inputs: Vec> = Vec::new(); + for (test, classification) in expanded.into_iter().zip(classified) { + if let Some(input) = self.checked(test, classification)? { + inputs.push(input); + } + } + Ok(inputs) + } + + /// One case's SQL with parameters, variables, declarations and macros expanded. + fn expanded<'a>( + &self, + file: &'a TestFileRow, + block: &'a TestBlockRow, + case: Option<&'a TestCaseRow>, + ) -> PyResult> { + let py: Python<'py> = self.py(); + let (file_path, relative_path, stem, _, contents, _) = file; + let (_, name, sql_body, mode, parameters, _, _) = block; + let test_name: &str = name.as_deref().unwrap_or(stem); + let resource: Bound<'py, PyAny> = self + .context + .classes + .visibility + .resource_identity + .call1((self.settings.test_kind.bind(py), test_name))?; + let declarations: Bound<'py, PyAny> = + scoped_declarations(self.context, file_path, &resource)?; + let parameter_sql: String = match case { + Some(case) => { + self.with_parameters(file_path, (test_name, parameters), sql_body, case)? + } + None => sql_body.clone(), + }; + let origin: FileOrigin<'_> = (file_path, relative_path, contents); + let tested: Vec = if mode == MODEL_MODE { + Vec::new() + } else { + let direct: SqlTestClassification = + self.raw_direct(¶meter_sql, (file, block), mode)?; + self.tested_names(&direct, mode, origin)? + }; + let macro_mode: bool = mode == MACRO_MODE; + let expansion_declarations: Bound<'py, PyAny> = if macro_mode { + let changes: Bound<'py, PyDict> = PyDict::new(py); + changes.set_item("consumer", py.None())?; + replaced(py, &declarations, &changes)?.into_bound(py) + } else { + declarations.clone() + }; + let expansion: AuthoredExpansion<'py> = expanded_authored_sql( + self.model_loop, + self.context, + (¶meter_sql, file_path), + &expansion_declarations, + )?; + let sql: String = checked_body( + self.model_loop, + self.context, + &expansion.sql, + &format!("SQL test '{test_name}'"), + )?; + let usages: Bound<'py, PyTuple> = if macro_mode { + self.macro_usages(¶meter_sql, &resource, &declarations)? + } else { + expansion.usages + }; + Ok(ExpandedTest { + file, + block, + case, + sql, + tested, + usages, + }) + } + + /// `expand_test_parameters`, then `SQL test ... declares unused parameters`. + fn with_parameters( + &self, + file_path: &str, + test: (&str, &[String]), + sql: &str, + case: &TestCaseRow, + ) -> PyResult { + let (test_name, parameters) = test; + let (_, case_name, values) = case; + let declared: Vec = values.iter().map(|(name, _)| name.clone()).collect(); + let owner: String = format!("SQL test '{test_name}' case '{case_name}' in '{file_path}'"); + let scan = parameter_references(sql, &declared, &owner); + let characters: Vec = sql.chars().collect(); + let mut expanded: String = String::with_capacity(sql.len()); + let mut cursor: usize = 0; + let mut used: HashSet<&str> = HashSet::new(); + for reference in &scan.references { + expanded.extend(&characters[cursor..reference.start]); + let value: &Py = values + .iter() + .find(|(name, _)| *name == reference.name) + .map(|(_, value)| value) + .ok_or_else(|| self.error(&reference.name))?; + expanded.push_str(&self.rendered_parameter( + (test_name, case_name), + &reference.name, + value, + )?); + used.insert(&reference.name); + cursor = reference.end; + } + expanded.extend(&characters[cursor..]); + if let Some(message) = scan.error { + return Err(self.error(&message)); + } + let unused: Vec<&str> = parameters + .iter() + .map(String::as_str) + .filter(|name| !used.contains(name)) + .collect(); + if !unused.is_empty() { + return Err(self.error(&format!( + "SQL test '{test_name}' declares unused parameters: {} in case '{case_name}'", + unused.join(", ") + ))); + } + Ok(expanded) + } + + /// `_render_parameter`: the adapter's rendering of one value, within the size limit. + fn rendered_parameter( + &self, + test: (&str, &str), + name: &str, + value: &Py, + ) -> PyResult { + let (test_name, case_name) = test; + let label: String = format!("SQL test '{test_name}' case '{case_name}' parameter '{name}'"); + let (text, error): (Option, Option) = self + .settings + .render_parameter + .bind(self.py()) + .call1((value,))? + .extract()?; + let problem: Option = match (&text, error) { + (_, Some(error)) => Some(error), + (Some(text), None) if text.len() > MAX_SQL_VALUE_SIZE => Some(format!( + "{label} rendered value is {} bytes; maximum is {MAX_SQL_VALUE_SIZE} bytes", + text.len() + )), + _ => None, + }; + if let Some(problem) = problem { + return Err(self.error(&format!( + "{label} could not be rendered by adapter '{}': {problem}", + self.settings.adapter_name + ))); + } + Ok(text.unwrap_or_default()) + } + + /// `_validate_raw_direct_logic_test_ctes`: the direct test before expansion, its P012s reported. + fn raw_direct( + &self, + sql: &str, + test: (&TestFileRow, &TestBlockRow), + mode: &str, + ) -> PyResult { + let (file, block) = test; + let request: SqlTestRequest = SqlTestRequest { + sql: sql.to_owned(), + file_label: file.1.clone(), + mode: mode.to_owned(), + raw: true, + authored: false, + scenario: false, + }; + let classification: SqlTestClassification = + match extract_sql_tests(&[request], &self.model_loop.syntax) { + Ok(mut classified) => classified.remove(0), + Err(failure) => return Err(self.located(&failure, file, (block, sql))), + }; + if let SqlTestClassification::Direct { + helpers, + actual, + expected, + invalid_calls: true, + .. + } = &classification + { + let origin: FileOrigin<'_> = (&file.0, &file.1, &file.4); + if self.authored_invalid_calls(file, block, &actual.0)? == 0 { + for cte in helpers.iter().chain([expected]) { + origin_references(self.context, &self.model_loop.syntax, &cte.1, origin)?; + } + } + } + Ok(classification) + } + + /// `report_authored_invalid_calls`: P012 for each call outside the actual CTE, located. + fn authored_invalid_calls( + &self, + file: &TestFileRow, + block: &TestBlockRow, + actual: &str, + ) -> PyResult { + let syntax = &self.model_loop.syntax; + let request: SqlTestRequest = SqlTestRequest { + sql: block.2.clone(), + file_label: file.1.clone(), + mode: MODEL_MODE.to_owned(), + raw: false, + authored: true, + scenario: false, + }; + let ctes: Vec<(String, usize, String)> = match extract_sql_tests(&[request], syntax) { + Ok(classified) => match classified.into_iter().next() { + Some(SqlTestClassification::Authored { ctes }) => ctes, + _ => Vec::new(), + }, + Err(failure) => { + return Err(helped_compile_error( + self.context.classes, + &failure.message, + failure.help.as_deref(), + )); + } + }; + let mut reported: usize = 0; + for (_, body_start, body) in ctes.iter().filter(|cte| cte.0 != actual) { + let scan = match extract_sql_references(body, syntax) { + ReferenceExtraction::Extracted(scan) => scan, + ReferenceExtraction::Failed(failure) => return Err(self.error(&failure.message)), + }; + for invalid in &scan.invalid_calls { + let Some(offset) = + authored_test_offset(&file.4, &block.2, block.6, body_start + invalid.start) + else { + return Ok(reported); + }; + report_invalid_call( + self.context, + invalid, + Some(offset), + (None, &file.1, &file.0, &file.4), + )?; + reported += 1; + } + } + Ok(reported) + } + + /// `_infer_tested_direct_logic_resource_names` for a macro, UDF or table function test. + fn tested_names( + &self, + classification: &SqlTestClassification, + mode: &str, + origin: FileOrigin<'_>, + ) -> PyResult> { + let relative_path: &str = origin.1; + let SqlTestClassification::Direct { actual, .. } = classification else { + return Err(self.error(&format!( + "SQL test file {relative_path} mode '{mode}' must define exactly one actual CTE \ + and exactly one expected CTE" + ))); + }; + if mode == MACRO_MODE { + let names: Vec = + top_level_macro_names(&actual.1).map_err(|message| self.error(message))?; + if names.is_empty() { + return Err(self.error(&format!( + "SQL test file {relative_path} mode 'macro' must call at least one macro in \ + __macro_actual__" + ))); + } + if let Some(unknown) = names + .iter() + .find(|name| !self.catalog.macros.contains(*name)) + { + return Err(self.error(&format!( + "SQL test file {relative_path} references unknown macro '@{unknown}'" + ))); + } + return Ok(names); + } + let references: Vec = + origin_references(self.context, &self.model_loop.syntax, &actual.1, origin)?; + let table_function: bool = mode == TABLE_FUNCTION_MODE; + if table_function { + let calls: Vec<(&str, &str, Option)> = references + .iter() + .map(|reference| { + ( + reference.kind, + reference.name.as_str(), + reference.call_argument_count, + ) + }) + .collect(); + if let Some(message) = table_function_arity_error( + &format!("SQL test file {relative_path}"), + &calls, + &self.table_function_arities, + ) { + return Err(self.error(&message)); + } + } + let kind: &str = if table_function { + TABLE_FUNCTION_KIND + } else { + UDF_KIND + }; + let names: Vec = unique( + references + .iter() + .filter(|reference| reference.kind == kind) + .map(|reference| reference.name.clone()), + ); + let (label, actual_cte) = if table_function { + ("table function", "__table_fn_actual__") + } else { + ("scalar UDF", "__udf_actual__") + }; + if names.is_empty() { + return Err(self.error(&format!( + "SQL test file {relative_path} mode '{mode}' must call at least one {label} in \ + {actual_cte}" + ))); + } + for name in &names { + if !self.functions.contains(name) { + return Err(self.error(&format!( + "SQL test file {relative_path} references unknown SQL function '{name}'" + ))); + } + let is_table: bool = self.catalog.table_functions.contains(name); + if !table_function && is_table { + return Err(self.error(&format!( + "SQL test file {relative_path} references table function '{name}' with \ + __udf(); use __table_fn() for table functions" + ))); + } + if table_function && !is_table { + return Err(self.error(&format!( + "SQL test file {relative_path} references scalar SQL function '{name}' with \ + __table_fn(); use __udf() for scalar UDFs" + ))); + } + } + Ok(names) + } + + /// `_macro_test_declaration_usages`: declaration, then macro, usages of a macro test. + fn macro_usages( + &self, + sql: &str, + resource: &Bound<'py, PyAny>, + declarations: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = self.py(); + let unique: Bound<'py, PyDict> = PyDict::new(py); + let scans = + sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references( + self.model_loop.python, + &[sql.to_owned()], + true, + ); + if let Some(stop) = scans.iter().find_map(|scan| scan.stop) { + let problem: &str = match stop { + DeclarationReferenceStop::UnclosedBlockComment => "unclosed block comment", + _ => "unclosed quoted string", + }; + return Err(self.error(&format!( + "Enum and constant expansion contains an {problem}" + ))); + } + let enum_visibility: Bound<'py, PyDict> = + declarations.getattr(ENUM_VISIBILITY)?.downcast_into()?; + let constant_visibility: Bound<'py, PyDict> = + declarations.getattr(CONSTANT_VISIBILITY)?.downcast_into()?; + for reference in scans.iter().flat_map(|scan| scan.references.iter()) { + let visibility: &Bound<'py, PyDict> = match reference.kind { + sqlbuild_sqltext::compiler::models::DeclarationReferenceKind::Enum => { + &enum_visibility + } + sqlbuild_sqltext::compiler::models::DeclarationReferenceKind::Constant => { + &constant_visibility + } + }; + let Some(records) = visibility.get_item(&reference.name)? else { + continue; + }; + for record in test_visibility(&records)? { + unique.set_item( + self.usage( + resource, + &record.getattr("declaration")?, + &record.getattr("through")?, + )?, + py.None(), + )?; + } + } + let macro_records: Bound<'py, PyDict> = + declarations.getattr(MACRO_RECORDS)?.downcast_into()?; + let macro_visibility: Bound<'py, PyDict> = + declarations.getattr(MACRO_VISIBILITY)?.downcast_into()?; + for name in top_level_macro_names(sql).map_err(|message| self.error(message))? { + let Some(record) = macro_records.get_item(&name)? else { + continue; + }; + let identity: Bound<'py, PyAny> = record.getattr("identity")?; + let visible: Vec> = match macro_visibility.get_item(&name)? { + Some(visibility) => test_visibility(&visibility)?, + None => Vec::new(), + }; + if visible.is_empty() { + unique.set_item( + self.usage(resource, &identity, &py.None().into_bound(py))?, + py.None(), + )?; + } + for record in visible { + unique.set_item( + self.usage(resource, &identity, &record.getattr("through")?)?, + py.None(), + )?; + } + } + PyTuple::new(py, unique.keys()) + } + + /// One runtime `UsageRecord` of `declaration` by `consumer`. + fn usage( + &self, + consumer: &Bound<'py, PyAny>, + declaration: &Bound<'py, PyAny>, + through: &Bound<'py, PyAny>, + ) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(self.py()); + fields.set_item("consumer", consumer)?; + fields.set_item("declaration", declaration)?; + fields.set_item("kind", self.settings.runtime_usage.bind(self.py()))?; + fields.set_item("through", through)?; + self.context.classes.usage_record.call((), Some(&fields)) + } + + /// `extract_expanded_sql_tests`: stored whole-file results, then the rest scanned and stored. + fn classified(&self, tests: &[ExpandedTest<'_, 'py>]) -> PyResult> { + let py: Python<'py> = self.py(); + let mut files: Vec<(&str, Vec)> = Vec::new(); + for (index, test) in tests.iter().enumerate() { + match files.iter_mut().find(|(label, _)| *label == test.file.1) { + Some((_, indexes)) => indexes.push(index), + None => files.push((&test.file.1, vec![index])), + } + } + let store: Option> = self + .settings + .scan_store + .as_ref() + .map(|store| store.bind(py).clone()); + let mut results: Vec> = tests.iter().map(|_| None).collect(); + let mut missing: Vec<(Vec, &Vec)> = Vec::new(); + for (label, indexes) in &files { + let mut parts: Vec = vec![ + self.settings.scan_key.0.clone(), + self.settings.scan_key.1.clone(), + (*label).to_owned(), + ]; + let mut modes: Vec<&str> = Vec::with_capacity(indexes.len()); + for index in indexes { + parts.push(tests[*index].sql.clone()); + parts.push(tests[*index].block.3.clone()); + modes.push(&tests[*index].block.3); + } + let stored: Option> = match &store { + Some(store) => store + .get() + .decoded(parts.clone(), |value| stored_sql_tests(value, &modes))?, + None => None, + }; + match stored { + Some(stored) => { + for (index, classification) in indexes.iter().zip(stored) { + results[*index] = Some(classification); + } + } + None => missing.push((parts, indexes)), + } + } + let mut pending: Vec = Vec::new(); + for (_, indexes) in &missing { + pending.extend(indexes.iter().copied()); + } + pending.sort_unstable(); + let requests: Vec = pending + .iter() + .map(|index| SqlTestRequest { + sql: tests[*index].sql.clone(), + file_label: tests[*index].file.1.clone(), + mode: tests[*index].block.3.clone(), + raw: false, + authored: false, + scenario: false, + }) + .collect(); + let classified: Vec = + extract_sql_tests(&requests, &self.model_loop.syntax).map_err(|failure| { + let test: &ExpandedTest<'_, 'py> = &tests[pending[failure.index]]; + self.located(&failure, test.file, (test.block, &test.sql)) + })?; + let mut by_index: HashMap = + pending.into_iter().zip(classified).collect(); + for (parts, indexes) in missing { + let mut items: Vec = Vec::with_capacity(indexes.len()); + for index in indexes { + items.push( + by_index + .remove(index) + .ok_or_else(|| self.error("SQL test was not classified"))?, + ); + } + if let Some(store) = &store { + let value: Vec = + serde_json::to_vec(&items).map_err(|error| self.error(&error.to_string()))?; + store.get().stored(parts, value)?; + } + for (index, item) in indexes.iter().zip(items) { + results[*index] = Some(item); + } + } + results + .into_iter() + .map(|result| result.ok_or_else(|| self.error("SQL test was not classified"))) + .collect() + } + + /// `located_extraction_error`: the error at its authored file line and column when exact. + fn located( + &self, + failure: &SqlTestExtractionFailure, + file: &TestFileRow, + test: (&TestBlockRow, &str), + ) -> PyErr { + let (block, sql) = test; + let offset: Option = failure + .token + .as_ref() + .filter(|_| sql == block.2) + .and_then(|(_, offset)| authored_test_offset(&file.4, &block.2, block.6, *offset)); + located_error(self.context, failure, (&file.1, &file.4), offset) + } + + /// Check one classified test; `None` when a P013 problem was reported instead. + fn checked( + &self, + test: ExpandedTest<'_, 'py>, + classification: SqlTestClassification, + ) -> PyResult>> { + let py: Python<'py> = self.py(); + let (file_path, relative_path, stem, test_file, contents, _) = test.file; + let (test_block, name, _, mode, _, _, _) = test.block; + let test_name: &str = name.as_deref().unwrap_or(stem); + let origin: FileOrigin<'_> = (file_path, relative_path, contents); + let payload: Bound<'py, PyAny> = match &classification { + SqlTestClassification::Model { + authored, + macro_mocks, + mock_models, + mock_sources, + mock_seeds, + mock_dbt_refs, + mock_table_functions, + expected, + expected_models, + assertions, + assertion_names, + .. + } => { + let targets: Vec = + assertion_targets(self.context, self.model_loop, assertions, origin)? + .into_iter() + .filter(|name| !mock_models.contains(name)) + .collect(); + if let Some(message) = unknown_test_target( + relative_path, + &self.catalog, + &TestTargets { + mock_models: mock_models.clone(), + mock_sources: mock_sources.clone(), + mock_seeds: mock_seeds.clone(), + mock_table_functions: mock_table_functions.clone(), + macro_mocks: macro_mocks.iter().map(|(name, _)| name.clone()).collect(), + expected_models: expected_models.clone(), + assertion_targets: targets.clone(), + }, + ) { + return Err(self.error(&message)); + } + let ctes: ModelTestCtes<'_> = ModelTestCtes { + authored, + expected, + assertions, + mock_models, + mock_sources, + mock_seeds, + mock_dbt_refs, + mock_table_functions, + }; + let patterns: PythonTextMatcher = PythonTextMatcher; + let problems: Vec = + self.checked_helpers(unresolvable_test_references( + &ctes, + &self.catalog.models, + &self.model_loop.syntax, + &patterns, + ))?; + if !problems.is_empty() { + for problem in &problems { + self.report_helper_problem(test.file, test.block, test_name, problem)?; + } + return Ok(None); + } + let reads: SqlTestReadFacts = self.checked_helpers(sql_test_reads( + &ctes, + &self.model_loop.syntax, + &patterns, + ))?; + if mode == MODEL_MODE + && expected_models.is_empty() + && targets.is_empty() + && reads.reference_target_model_names.is_empty() + { + let problem: SqlTestHelperProblem = + self.checked_helpers(test_without_target_model( + &ctes, + test_name, + &self.model_loop.syntax, + &patterns, + ))?; + self.report_helper_problem(test.file, test.block, test_name, &problem)?; + return Ok(None); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("authored_ctes", self.ctes(authored)?)?; + let mocks: Bound<'py, PyDict> = PyDict::new(py); + for (name, sql) in macro_mocks { + mocks.set_item(name, sql)?; + } + fields.set_item("macro_mocks", mocks)?; + fields.set_item("mock_model_names", PyTuple::new(py, mock_models)?)?; + fields.set_item("mock_source_names", PyTuple::new(py, mock_sources)?)?; + fields.set_item("mock_seed_names", PyTuple::new(py, mock_seeds)?)?; + fields.set_item("mock_dbt_ref_names", PyTuple::new(py, mock_dbt_refs)?)?; + fields.set_item( + "mock_table_function_names", + PyTuple::new(py, mock_table_functions)?, + )?; + fields.set_item("expected_ctes", self.ctes(expected)?)?; + fields.set_item("expected_model_names", PyTuple::new(py, expected_models)?)?; + fields.set_item("assertion_ctes", self.ctes(assertions)?)?; + fields.set_item("assertion_names", PyTuple::new(py, assertion_names)?)?; + fields.set_item("assertion_target_model_names", PyTuple::new(py, targets)?)?; + fields.set_item( + "read_helper_names", + PyTuple::new(py, reads.read_helper_names)?, + )?; + fields.set_item( + "reference_target_model_names", + PyTuple::new(py, reads.reference_target_model_names)?, + )?; + self.settings + .model_payload + .bind(py) + .call((), Some(&fields))? + } + SqlTestClassification::Direct { + mode, + helpers, + actual, + expected, + .. + } => { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("mode", self.mode(mode)?)?; + fields.set_item("helper_ctes", self.ctes(helpers)?)?; + fields.set_item("actual_cte", self.cte(actual)?)?; + fields.set_item("expected_cte", self.cte(expected)?)?; + fields.set_item("tested_resource_names", PyTuple::new(py, &test.tested)?)?; + self.settings + .direct_payload + .bind(py) + .call((), Some(&fields))? + } + SqlTestClassification::Authored { .. } => { + return Err(self.error("native SQL test extraction returned an invalid kind")); + } + }; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("test_file", test_file.bind(py))?; + fields.set_item("test_block", test_block.bind(py))?; + fields.set_item("sql_body", &test.sql)?; + fields.set_item("mode", self.mode(mode)?)?; + fields.set_item("payload", payload)?; + fields.set_item("declaration_usages", &test.usages)?; + fields.set_item("parent_name", test_name)?; + let block: &Bound<'py, PyAny> = test_block.bind(py); + fields.set_item("parameter_schema", block.getattr("parameters")?)?; + if let Some((case, case_name, _)) = test.case { + let case: &Bound<'py, PyAny> = case.bind(py); + fields.set_item("case_name", case_name)?; + fields.set_item("case_index", case.getattr("case_index")?)?; + fields.set_item("parameter_values", case.getattr("values")?)?; + } + Ok(Some( + self.settings + .test_input + .bind(py) + .call((), Some(&fields))? + .unbind(), + )) + } + + fn checked_helpers( + &self, + outcome: Result>, + ) -> PyResult { + outcome.map_err(|failure| match failure { + SqlTestCheckFailure::Internal(message) | SqlTestCheckFailure::Scan(message) => { + self.error(&message) + } + SqlTestCheckFailure::Matcher(never) => match never {}, + }) + } + + /// Report one P013 problem at its call, else its CTE header, else the test block. + fn report_helper_problem( + &self, + file: &TestFileRow, + block: &TestBlockRow, + test_name: &str, + problem: &SqlTestHelperProblem, + ) -> PyResult<()> { + let py: Python<'py> = self.py(); + let located: CteLocation = sql_test_cte_location( + &file.4, + &block.2, + &problem.cte_name, + problem.call.as_deref(), + ); + let (line, column) = (located.line, located.column); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("path", file.3.bind(py).getattr("relative_path")?)?; + arguments.set_item("line", line)?; + arguments.set_item("column", column)?; + arguments.set_item("end_line", located.end_line)?; + arguments.set_item("end_column", located.end_column)?; + let location: Bound<'py, PyAny> = self + .settings + .source_location + .bind(py) + .call((), Some(&arguments))?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", "error")?; + fields.set_item("code", HELPER_REFERENCE_CODE)?; + fields.set_item("message", &problem.message)?; + fields.set_item("resource_type", SQL_TEST_RESOURCE)?; + fields.set_item("resource_name", test_name)?; + fields.set_item("location", location)?; + fields.set_item("help", &problem.help)?; + let key: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + HELPER_REFERENCE_CODE.to_owned(), + file.1.replace('\\', "/"), + format!("{line:09}:{column:09}"), + problem.message.clone(), + ], + )?; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("key", key)?; + arguments.set_item( + "diagnostic", + self.context + .classes + .compiler_diagnostic + .call((), Some(&fields))?, + )?; + self.context + .classes + .report_diagnostic + .call((), Some(&arguments))?; + Ok(()) + } + + fn mode(&self, mode: &str) -> PyResult> { + self.settings + .modes + .bind(self.py()) + .get_item(mode)? + .ok_or_else(|| self.error(&format!("unsupported SQL test mode '{mode}'"))) + } + + fn cte(&self, cte: &SqlTestCteText) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(self.py()); + fields.set_item("name", &cte.0)?; + fields.set_item("sql_body", &cte.1)?; + self.settings + .test_cte + .bind(self.py()) + .call((), Some(&fields)) + } + + fn ctes(&self, ctes: &[SqlTestCteText]) -> PyResult> { + let records: Vec> = ctes + .iter() + .map(|cte| self.cte(cte)) + .collect::>()?; + PyTuple::new(self.py(), records) + } +} + +/// `usage_visibility` for a test or scenario consumer: relationship grants, else every record. +fn test_visibility<'py>(records: &Bound<'py, PyAny>) -> PyResult>> { + let all: Vec> = records.try_iter()?.collect::>()?; + let mut granted: Vec> = Vec::new(); + for record in &all { + if !record.getattr("through")?.is_none() { + granted.push(record.clone()); + } + } + Ok(if granted.is_empty() { all } else { granted }) +} + +/// `find_macro_call_names`: top-level macro call names, declaration calls excluded. +fn top_level_macro_names(sql: &str) -> Result, &'static str> { + Ok(macro_call_names(active_python_text(), sql, false) + .map_err(|error| error.message())? + .into_iter() + .filter(|name| !DECLARATION_CALLS.contains(&name.as_str())) + .collect()) +} + +/// The SQL with an omitted trailing `SELECT 1` completed, free of cursor intrinsics. +pub(crate) fn checked_body( + model_loop: &NativeModelLoop, + context: &LoopContext<'_, '_>, + sql: &str, + label: &str, +) -> PyResult { + let completed: String = match omitted_select_offset(sql, &model_loop.syntax) { + OmittedSelect::At(offset) => { + let mut characters = sql.chars(); + let head: String = characters.by_ref().take(offset).collect(); + format!("{head}{OMITTED_CEREMONIAL_SELECT}{}", characters.as_str()) + } + OmittedSelect::Absent => sql.to_owned(), + }; + match intrinsic_free( + model_loop.python, + &completed, + &model_loop.reserved_markers, + label, + ) { + IntrinsicCheck::Free => Ok(completed), + IntrinsicCheck::Rejected(message) => Err(compile_error(context.classes, &message, None)), + } +} + +/// `extract_assertion_target_model_names`: unique `__ref` targets of the assertions, in order. +pub(crate) fn assertion_targets( + context: &LoopContext<'_, '_>, + model_loop: &NativeModelLoop, + assertions: &[SqlTestCteText], + origin: FileOrigin<'_>, +) -> PyResult> { + let mut targets: Vec = Vec::new(); + for cte in assertions { + for reference in origin_references(context, &model_loop.syntax, &cte.1, origin)? { + if reference.kind == REF_KIND { + targets.push(reference.name); + } + } + } + Ok(unique(targets)) +} + +/// The names of `references` of `kind`, as `__source` reads list them. +pub(crate) fn source_names(references: &[SqlReference]) -> Vec { + references + .iter() + .filter(|reference| reference.kind == SOURCE_KIND) + .map(|reference| reference.name.clone()) + .collect() +} + +/// `_located_error`: the message with ` --> path:line:column` when the offset is known. +pub(crate) fn located_error( + context: &LoopContext<'_, '_>, + failure: &SqlTestExtractionFailure, + file: (&str, &str), + offset: Option, +) -> PyErr { + let (relative_path, contents) = file; + let message: String = match offset { + Some(offset) => { + let position: TextPosition = text_position(contents, offset); + format!( + "{}\n --> {}:{}:{}", + failure.message, + relative_path.replace('\\', "/"), + position.line, + position.column + ) + } + None => failure.message.clone(), + }; + helped_compile_error(context.classes, &message, failure.help.as_deref()) +} + +pub(crate) fn unique(items: impl IntoIterator) -> Vec { + let mut seen: HashSet = HashSet::new(); + items + .into_iter() + .filter(|item| seen.insert(item.clone())) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs new file mode 100644 index 0000000000..d9e7695d33 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs @@ -0,0 +1,122 @@ +//! Python values as the JSON `orjson.dumps(value, option=OPT_SORT_KEYS, default=str)` writes. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, + PyStringMethods, PyTuple, +}; +use serde_json::{Map, Number, Value}; + +use crate::bindings::_helpers::boundary::panics::value_error; + +const DATACLASS_FIELDS: &str = "__dataclass_fields__"; +const INSTANCE_DICT: &str = "__dict__"; +const ENUM_MODULE: &str = "enum"; +const ENUM_TYPE: &str = "Enum"; +const DATETIME_MODULE: &str = "datetime"; +const DATE_TYPE: &str = "date"; +const TIME_TYPE: &str = "time"; +const ISOFORMAT: &str = "isoformat"; +const ENUM_VALUE: &str = "value"; +const PRIVATE_PREFIX: &str = "_"; +const ORJSON_WIDE_INTEGER: &str = "Integer exceeds 64-bit range"; + +/// orjson's value for `value`: enums by value, dataclasses by public attribute, dates in ISO form. +pub(crate) fn plain_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(Value::Null); + } + if let Ok(flag) = value.downcast::() { + return Ok(Value::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return integer_value(value); + } + if value.is_instance_of::() { + return Ok(Number::from_f64(value.extract::()?).map_or(Value::Null, Value::Number)); + } + if let Ok(text) = value.downcast::() { + return Ok(Value::String(text.to_str()?.to_owned())); + } + if value.is_instance_of::() || value.is_instance_of::() { + return value + .try_iter()? + .map(|item| plain_value(&item?)) + .collect::>>() + .map(Value::Array); + } + if let Ok(mapping) = value.downcast::() { + return object_value(mapping); + } + object_like_value(value) +} + +/// Enums, dataclasses and dates as orjson writes them; anything else through `default=str`. +fn object_like_value(value: &Bound<'_, PyAny>) -> PyResult { + let py = value.py(); + if value.is_instance(&py.import(ENUM_MODULE)?.getattr(ENUM_TYPE)?)? { + return plain_value(&value.getattr(ENUM_VALUE)?); + } + if value.hasattr(DATACLASS_FIELDS)? && !value.is_instance_of::() { + return dataclass_value(value); + } + let datetime = py.import(DATETIME_MODULE)?; + if value.is_instance(&datetime.getattr(DATE_TYPE)?)? + || value.is_instance(&datetime.getattr(TIME_TYPE)?)? + { + return Ok(Value::String( + value.call_method0(ISOFORMAT)?.str()?.to_str()?.to_owned(), + )); + } + Ok(Value::String(value.str()?.to_str()?.to_owned())) +} + +/// A dataclass's public instance attributes, or its public fields when it has slots. +fn dataclass_value(value: &Bound<'_, PyAny>) -> PyResult { + let mut entries: Vec<(String, Value)> = Vec::new(); + if let Ok(attributes) = value.getattr(INSTANCE_DICT) { + for (key, item) in attributes.downcast::()?.iter() { + let name: String = key.str()?.to_str()?.to_owned(); + if !name.starts_with(PRIVATE_PREFIX) { + entries.push((name, plain_value(&item)?)); + } + } + } else { + for key in value.getattr(DATACLASS_FIELDS)?.try_iter()? { + let name: String = key?.str()?.to_str()?.to_owned(); + if !name.starts_with(PRIVATE_PREFIX) { + let item = value.getattr(name.as_str())?; + entries.push((name, plain_value(&item)?)); + } + } + } + Ok(sorted_object(entries)) +} + +/// A Python integer within 64 bits; orjson rejects wider integers with the same message. +fn integer_value(value: &Bound<'_, PyAny>) -> PyResult { + if let Ok(signed) = value.extract::() { + return Ok(Value::from(signed)); + } + match value.extract::() { + Ok(unsigned) => Ok(Value::from(unsigned)), + Err(_) => Err(value_error(ORJSON_WIDE_INTEGER)), + } +} + +/// A string-keyed dictionary with its keys sorted, as `OPT_SORT_KEYS` encodes it. +pub(crate) fn object_value(mapping: &Bound<'_, PyDict>) -> PyResult { + let mut entries: Vec<(String, Value)> = Vec::new(); + for (key, item) in mapping.iter() { + let Ok(key) = key.downcast::() else { + return Err(value_error("Dict key must be str")); + }; + entries.push((key.to_str()?.to_owned(), plain_value(&item)?)); + } + Ok(sorted_object(entries)) +} + +fn sorted_object(mut entries: Vec<(String, Value)>) -> Value { + entries.sort_by(|left, right| left.0.cmp(&right.0)); + Value::Object(entries.into_iter().collect::>()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs new file mode 100644 index 0000000000..3f6575ad8f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/dynamic_names.rs @@ -0,0 +1,32 @@ +//! Compiled dynamic column family `name_pattern`s for matching runtime column names. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_model_config::name_patterns::main::compile_name_pattern::compile_name_pattern; +use sqlbuild_model_config::name_patterns::models::NamePattern; + +/// A `name_pattern` compiled to match whole column names; an invalid pattern raises `ValueError`. +#[pyclass(name = "NamePattern", module = "sqlbuild._native", frozen)] +struct PyNamePattern { + pattern: NamePattern, +} + +#[pymethods] +impl PyNamePattern { + #[new] + fn new(pattern: &str) -> PyResult { + compile_name_pattern(pattern) + .map(|pattern| Self { pattern }) + .map_err(|error| PyValueError::new_err(format!("invalid name_pattern: {error}"))) + } + + /// Whether `column_name` wholly matches the pattern. + fn matches(&self, column_name: &str) -> bool { + self.pattern.matches(column_name) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs new file mode 100644 index 0000000000..8ec58220c6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs @@ -0,0 +1,346 @@ +//! Effective target, vars, connection and settings precedence over the loaded config objects. + +use std::hash::{Hash, Hasher}; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, PyStringMethods, Python, +}; +use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyListMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_config::effective::errors::{EffectiveConfigError, MessagePart}; +use sqlbuild_config::effective::main::default_connection_name::default_connection_name; +use sqlbuild_config::effective::main::effective_connection::effective_connection; +use sqlbuild_config::effective::main::effective_settings::effective_settings; +use sqlbuild_config::effective::main::effective_target_name::effective_target_name; +use sqlbuild_config::effective::main::effective_vars::effective_vars; +use sqlbuild_config::effective::main::merged_target::merged_target; +use sqlbuild_config::effective::main::target_fields::target_fields; +use sqlbuild_config::effective::models::{ + ConnectionChoice, ConnectionLayers, FieldValue, MergeRule, TargetChoice, TargetField, + TargetLayer, +}; +use sqlbuild_config::effective::types::Entries; + +const UTF8: &str = "utf-8"; +const SURROGATE_PASS: &str = "surrogatepass"; + +/// A `str` key compared exactly: by its UTF-8 bytes, lone surrogates passed through. +#[derive(Clone)] +struct Key<'py> { + bytes: Vec, + handle: Bound<'py, PyString>, +} + +impl PartialEq for Key<'_> { + fn eq(&self, other: &Self) -> bool { + self.bytes == other.bytes + } +} + +impl Eq for Key<'_> {} + +impl Hash for Key<'_> { + fn hash(&self, state: &mut H) { + self.bytes.hash(state); + } +} + +impl PartialOrd for Key<'_> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Key<'_> { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.bytes.cmp(&other.bytes) + } +} + +type Value<'py> = Bound<'py, PyAny>; +type Layer<'py> = TargetLayer, Value<'py>>; +type Pairs<'py> = Entries, Value<'py>>; +/// `(group, name, value)` of each merged `TargetConfig` field. +type FieldRow<'py> = (Option<&'static str>, &'static str, Value<'py>); +/// A result, else the precedence failure message. +type Outcome<'py, T> = (Option, Option>); + +fn key<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let handle: Bound<'py, PyString> = value.downcast::()?.clone(); + if let Ok(text) = handle.to_str() { + return Ok(Key { + bytes: text.as_bytes().to_vec(), + handle, + }); + } + let bytes: Vec = handle + .call_method1("encode", (UTF8, SURROGATE_PASS))? + .downcast_into::()? + .as_bytes() + .to_vec(); + Ok(Key { bytes, handle }) +} + +fn optional_key<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + if value.is_none() { + return Ok(None); + } + key(value).map(Some) +} + +fn pairs<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { + mapping + .downcast::()? + .iter() + .map(|(name, value)| Ok((key(&name)?, value))) + .collect() +} + +fn keys<'py>(mapping: &Bound<'py, PyAny>) -> PyResult>> { + mapping + .downcast::()? + .keys() + .iter() + .map(|name| key(&name)) + .collect() +} + +fn dictionary<'py>(py: Python<'py>, entries: Pairs<'py>) -> PyResult> { + let dictionary: Bound<'py, PyDict> = PyDict::new(py); + for (name, value) in entries { + dictionary.set_item(name.handle, value)?; + } + Ok(dictionary) +} + +fn field_owner<'py>( + target: &Bound<'py, PyAny>, + field: &TargetField, +) -> PyResult> { + match field.group { + Some(group) => target.getattr(group), + None => Ok(target.clone()), + } +} + +fn field_value<'py>( + target: &Bound<'py, PyAny>, + field: &TargetField, +) -> PyResult, Value<'py>>> { + let value: Bound<'py, PyAny> = field_owner(target, field)?.getattr(field.name)?; + match (field.rule, value.is_none()) { + (MergeRule::Overlay { .. }, true) => Ok(FieldValue::Scalar(None)), + (MergeRule::Overlay { .. }, false) => Ok(FieldValue::Scalar(Some(value))), + (_, true) => Ok(FieldValue::Map(None)), + (_, false) => Ok(FieldValue::Map(Some(pairs(&value)?))), + } +} + +/// One target dataclass as a layer of `TARGET_FIELDS`. +fn target_layer<'py>(target: &Bound<'py, PyAny>) -> PyResult> { + Ok(TargetLayer { + fields: target_fields() + .iter() + .map(|field| field_value(target, field)) + .collect::>()?, + }) +} + +fn field_position(name: &str) -> usize { + target_fields() + .iter() + .position(|field| field.group.is_none() && field.name == name) + .unwrap_or_default() +} + +fn is_set(value: &FieldValue, Value<'_>>) -> bool { + match value { + FieldValue::Scalar(value) => value.is_some(), + FieldValue::Map(value) => value.as_ref().is_some_and(|entries| !entries.is_empty()), + } +} + +/// The failure message as a Python `str`, authored names inserted unchanged. +fn message<'py>( + py: Python<'py>, + error: &EffectiveConfigError>, +) -> PyResult> { + let parts: Vec> = error + .message_parts() + .into_iter() + .map(|part| match part { + MessagePart::Text(text) => PyString::new(py, text), + MessagePart::Name(name) => name.handle.clone(), + }) + .collect(); + Ok(PyString::new(py, "") + .call_method1("join", (parts,))? + .downcast_into::()?) +} + +/// `resolve_target_name`: the effective target, `None`, or the unknown-target message. +#[pyfunction] +fn effective_config_target_name<'py>( + py: Python<'py>, + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + selected: &Bound<'py, PyAny>, +) -> PyResult>> { + let mut defined: Vec> = keys(&project_config.getattr("targets")?)?; + defined.extend(keys(&local_config.getattr("targets")?)?); + let choice: TargetChoice> = TargetChoice { + selected: optional_key(selected)?, + local: optional_key(&local_config.getattr("target")?)?, + default: optional_key(&project_config.getattr("default_target")?)?, + defined, + }; + Ok(match effective_target_name(choice) { + Ok(name) => (name.map(|name| name.handle), None), + Err(error) => (None, Some(message(py, &error)?)), + }) +} + +/// `resolve_target_config`: each merged `TargetConfig` field, or the ambiguous-connection message. +#[pyfunction] +fn effective_config_target_fields<'py>( + py: Python<'py>, + configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), + target_name: &Bound<'py, PyAny>, + default_target: &Bound<'py, PyAny>, +) -> PyResult>>> { + let (project_config, local_config) = configs; + let name: Key<'py> = key(target_name)?; + let project_target: Option> = project_config + .getattr("targets")? + .downcast::()? + .get_item(&name.handle)?; + let local_target: Option> = local_config + .getattr("targets")? + .downcast::()? + .get_item(&name.handle)?; + let local: Option> = local_target.as_ref().map(target_layer).transpose()?; + let mut merged: Layer<'py> = merged_target( + target_layer(project_target.as_ref().unwrap_or(default_target))?, + local.as_ref(), + ); + let connection_name: usize = field_position("connection_name"); + let choice: ConnectionChoice> = ConnectionChoice { + target: name, + selected: is_set(&merged.fields[connection_name]) + || is_set(&merged.fields[field_position("connection")]) + || project_config.getattr("connection")?.is_truthy()? + || local_config.getattr("connection")?.is_truthy()?, + project_names: keys(&project_config.getattr("connections")?)?, + local_names: keys(&local_config.getattr("connections")?)?, + }; + match default_connection_name(choice) { + Ok(Some(only)) => { + merged.fields[connection_name] = FieldValue::Scalar(Some(only.handle.into_any())); + } + Ok(None) => {} + Err(error) => return Ok((None, Some(message(py, &error)?))), + } + let mut rows: Vec> = Vec::new(); + for (field, value) in target_fields().iter().zip(merged.fields) { + let value: Value<'py> = match value { + FieldValue::Scalar(value) => value.unwrap_or_else(|| py.None().into_bound(py)), + FieldValue::Map(None) => py.None().into_bound(py), + FieldValue::Map(Some(entries)) => dictionary(py, entries)?.into_any(), + }; + rows.push((field.group, field.name, value)); + } + Ok((Some(rows), None)) +} + +/// `build_effective_vars` before template expansion. +#[pyfunction] +fn effective_config_vars<'py>( + py: Python<'py>, + configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), + target_config: &Bound<'py, PyAny>, + cli_vars: &Bound<'py, PyAny>, +) -> PyResult> { + let (project_config, local_config) = configs; + let project: Pairs<'py> = pairs(&project_config.getattr("vars")?)?; + let target: Pairs<'py> = if target_config.is_none() { + Vec::new() + } else { + pairs(&target_config.getattr("vars")?)? + }; + let local: Pairs<'py> = pairs(&local_config.getattr("vars")?)?; + let cli: Pairs<'py> = pairs(cli_vars)?; + dictionary(py, effective_vars(&[&project, &target, &local, &cli])) +} + +fn named<'py>(mapping: &Bound<'py, PyAny>) -> PyResult, Pairs<'py>>> { + mapping + .downcast::()? + .iter() + .map(|(name, connection)| Ok((key(&name)?, pairs(&connection)?))) + .collect() +} + +/// `build_effective_connection` before template expansion, or the unknown-connection message. +#[pyfunction] +fn effective_config_connection<'py>( + py: Python<'py>, + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, +) -> PyResult>> { + let target: Option<(Option>, Pairs<'py>)> = if target_config.is_none() { + None + } else { + Some(( + optional_key(&target_config.getattr("connection_name")?)?, + pairs(&target_config.getattr("connection")?)?, + )) + }; + let layers: ConnectionLayers, Value<'py>> = ConnectionLayers { + project: pairs(&project_config.getattr("connection")?)?, + target, + project_named: named(&project_config.getattr("connections")?)?, + local_named: named(&local_config.getattr("connections")?)?, + local: pairs(&local_config.getattr("connection")?)?, + }; + Ok(match effective_connection(layers) { + Ok(connection) => (Some(dictionary(py, connection)?), None), + Err(error) => (None, Some(message(py, &error)?)), + }) +} + +/// `build_effective_settings`: setting values by field name, local overrides applied. +#[pyfunction] +fn effective_config_settings<'py>( + py: Python<'py>, + settings: (Bound<'py, PyAny>, Bound<'py, PyAny>), + overrides: &Bound<'py, PyAny>, + names: Vec>, +) -> PyResult> { + let read = |settings: &Bound<'py, PyAny>| -> PyResult> { + names + .iter() + .map(|name| Ok((key(name.as_any())?, settings.getattr(name)?))) + .collect() + }; + let overridden: Vec> = overrides + .try_iter()? + .map(|name| key(&name?)) + .collect::>()?; + let (project_settings, local_settings) = settings; + let settings: Pairs<'py> = effective_settings( + read(&project_settings)?, + &read(&local_settings)?, + &overridden, + ); + dictionary(py, settings) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(effective_config_target_name, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_target_fields, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_vars, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_connection, module)?)?; + module.add_function(wrap_pyfunction!(effective_config_settings, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs index abe28656cc..51adb352dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -1,2 +1,7 @@ +pub(crate) mod dynamic_names; +pub(crate) mod effective_config; pub(crate) mod model_contracts; +pub(crate) mod project_config; pub(crate) mod registration; +pub(crate) mod resource_identities; +pub(crate) mod sql_values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs index 7f41ace49c..71cf954296 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs @@ -1,7 +1,11 @@ //! Model column contracts and promotion conflicts for the preview compiler engine. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiled_project::main::project_models::project_models; +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, DynamicProofFact, ModelAnalysisFact, ModelSchemaFact, +}; use sqlbuild_analysis::contracts::main::evaluate_model_contracts::evaluate_model_contracts; use sqlbuild_analysis::contracts::main::promotion_conflicts::promotion_conflicts; use sqlbuild_analysis::contracts::models::{ @@ -13,25 +17,10 @@ use sqlbuild_analysis::contracts::models::{ use sqlbuild_analysis::type_system::errors::TypeNormalizationError; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::type_system::normalization::normalization_error; use crate::bindings::types::CompilerDetach; -/// `(name, type, not_null, declared_in_named_schema)`. -type DeclaredRow = (String, Option, bool, bool); -/// `(columns, dynamic_columns, type_enforcement, named_schema)`. -type SchemaRow = (Vec, Vec<(String, String)>, bool, bool); -/// `(output_proven, failure_reason, [(family, inferred_type)])`. -type ProofRow = (bool, Option, Vec<(String, Option)>); -/// `(name, contract, schema, inferred_columns, fast_lineage_has_star, proof, unchecked)`. -type ModelRow = ( - String, - Option, - Option, - Option, bool)>>, - bool, - Option, - Vec, -); /// `(code, is_error, message, column, declared_index, output_column, related, help)`. type DiagnosticRow = ( &'static str, @@ -43,53 +32,135 @@ type DiagnosticRow = ( Option<(String, String)>, String, ); -/// `(name, contract, materialized, incremental_mode)`. -type PromotionModelRow = (String, Option, Option, Option); -/// Evaluate every model's column contract, raising Python's type normalization error. +/// Evaluate the column contract of every model the compile's owner recorded, in its order. #[pyfunction] -fn evaluate_native_model_contracts( +fn evaluate_project_model_contracts( py: Python<'_>, - request: (String, bool, Vec), -) -> PyResult>> { - let (dialect, implicit_column_contracts, models) = request; + settings: (String, bool), + project: PyRef<'_, NativeCompiledProject>, +) -> PyResult)>> { + let (dialect, implicit_column_contracts) = settings; + let models: &[CompiledModelFacts] = project_models(project.facts()); let request = ContractRequest { dialect, implicit_column_contracts, - models: models.into_iter().map(contract_model).collect(), + models: models.iter().map(fact_contract_model).collect(), }; let outcomes: Result>, TypeNormalizationError> = py .compiler_detach(|| Ok(evaluate_model_contracts(&request))) .map_err(compiler_error)?; - Ok(outcomes - .map_err(|error| normalization_error(py, &error))? - .into_iter() - .map(|diagnostics| diagnostics.into_iter().map(diagnostic_row).collect()) + let outcomes: Vec> = + outcomes.map_err(|error| normalization_error(py, &error))?; + Ok(models + .iter() + .zip(outcomes) + .map(|(model, diagnostics)| { + ( + model.name.clone(), + diagnostics.into_iter().map(diagnostic_row).collect(), + ) + }) .collect()) } -/// The K011 conflicts of enforced contracts under immediate table promotion. +/// One model's contract inputs, read from its retained facts. +fn fact_contract_model(model: &CompiledModelFacts) -> ContractModel { + let analysis: Option<&ModelAnalysisFact> = model.analysis.as_ref(); + ContractModel { + name: model.name.clone(), + contract: model.config.contract.clone(), + schema: model.schema.as_ref().map(fact_contract_schema), + inferred_columns: analysis + .filter(|analysis| analysis.inferred) + .map(fact_inferred_columns), + fast_lineage_has_star: analysis.is_some_and(|analysis| analysis.fast_lineage_has_star), + dynamic_proof: analysis + .and_then(|analysis| analysis.dynamic_proof.as_ref()) + .map(fact_dynamic_proof), + unchecked_output_columns: analysis + .map(|analysis| analysis.unchecked_output_columns.clone()) + .unwrap_or_default(), + } +} + +fn fact_contract_schema(schema: &ModelSchemaFact) -> ContractSchema { + let mut columns: Vec = Vec::with_capacity(schema.columns.len()); + for column in &schema.columns { + columns.push(DeclaredColumn { + name: column.name.clone(), + declared_type: column.data_type.clone(), + not_null: column.nullable == Some(false), + declared_in_named_schema: schema.named_schema && column.declared_outside_model, + }); + } + let mut dynamic_columns: Vec = + Vec::with_capacity(schema.dynamic_columns.len()); + for family in &schema.dynamic_columns { + dynamic_columns.push(DeclaredColumnFamily { + name: family.name.clone(), + declared_type: family.data_type.clone(), + }); + } + ContractSchema { + columns, + dynamic_columns, + type_enforcement: schema.type_enforcement, + named_schema: schema.named_schema, + } +} + +fn fact_inferred_columns(analysis: &ModelAnalysisFact) -> Vec { + let mut columns: Vec = + Vec::with_capacity(analysis.inferred_columns.len()); + for (index, (name, inferred_type)) in analysis.inferred_columns.iter().enumerate() { + columns.push(InferredOutputColumn { + name: name.clone(), + inferred_type: inferred_type.clone(), + nullable: analysis + .nullable_columns + .get(index) + .copied() + .unwrap_or(false), + }); + } + columns +} + +fn fact_dynamic_proof(proof: &DynamicProofFact) -> DynamicContractProof { + DynamicContractProof { + output_proven: proof.output_proven, + failure_reason: proof.failure_reason.clone(), + families: proof.named_families.clone(), + } +} + +/// `(model name, model path, code, message, help)`. +type ConflictRow = (String, String, &'static str, String, String); + +/// The K011 conflicts of enforced contracts under immediate table promotion, by model name. #[pyfunction] -fn native_promotion_conflicts( +fn project_promotion_conflicts( py: Python<'_>, - request: (Option, String, String, Vec), -) -> PyResult> { - let (explicit_mode, adapter_default, settings_file, models) = request; + settings: (Option, String, String), + project: PyRef<'_, NativeCompiledProject>, +) -> PyResult> { + let (explicit_mode, adapter_default, settings_file) = settings; + let models: &[CompiledModelFacts] = project_models(project.facts()); + let mut rows: Vec = Vec::with_capacity(models.len()); + for model in models { + rows.push(PromotionModel { + name: model.name.clone(), + contract: model.config.contract.clone(), + materialized: model.config.materialized.clone(), + incremental_mode: model.config.incremental_mode.clone(), + }); + } let request = PromotionRequest { explicit_mode, adapter_default, settings_file, - models: models - .into_iter() - .map( - |(name, contract, materialized, incremental_mode)| PromotionModel { - name, - contract, - materialized, - incremental_mode, - }, - ) - .collect(), + models: rows, }; let conflicts: Vec = py .compiler_detach(|| Ok(promotion_conflicts(&request))) @@ -97,8 +168,10 @@ fn native_promotion_conflicts( Ok(conflicts .into_iter() .map(|conflict| { + let model: &CompiledModelFacts = &models[conflict.model_index]; ( - conflict.model_index, + model.name.clone(), + model.relative_path.clone(), conflict.code, conflict.message, conflict.help, @@ -107,64 +180,6 @@ fn native_promotion_conflicts( .collect()) } -fn contract_model( - (name, contract, schema, inferred, has_star, proof, unchecked): ModelRow, -) -> ContractModel { - ContractModel { - name, - contract, - schema: schema.map(contract_schema), - inferred_columns: inferred.map(inferred_columns), - fast_lineage_has_star: has_star, - dynamic_proof: proof.map(|(output_proven, failure_reason, families)| { - DynamicContractProof { - output_proven, - failure_reason, - families, - } - }), - unchecked_output_columns: unchecked, - } -} - -fn inferred_columns(columns: Vec<(String, Option, bool)>) -> Vec { - columns - .into_iter() - .map(|(name, inferred_type, nullable)| InferredOutputColumn { - name, - inferred_type, - nullable, - }) - .collect() -} - -fn contract_schema( - (columns, dynamic_columns, type_enforcement, named_schema): SchemaRow, -) -> ContractSchema { - ContractSchema { - columns: columns - .into_iter() - .map( - |(name, declared_type, not_null, declared_in_named_schema)| DeclaredColumn { - name, - declared_type, - not_null, - declared_in_named_schema, - }, - ) - .collect(), - dynamic_columns: dynamic_columns - .into_iter() - .map(|(name, declared_type)| DeclaredColumnFamily { - name, - declared_type, - }) - .collect(), - type_enforcement, - named_schema, - } -} - fn diagnostic_row(diagnostic: ContractDiagnostic) -> DiagnosticRow { let (declared_index, output_column) = diagnostic.location.into_parts(); ( @@ -182,7 +197,7 @@ fn diagnostic_row(diagnostic: ContractDiagnostic) -> DiagnosticRow { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(evaluate_native_model_contracts, module)?)?; - module.add_function(wrap_pyfunction!(native_promotion_conflicts, module)?)?; + module.add_function(wrap_pyfunction!(evaluate_project_model_contracts, module)?)?; + module.add_function(wrap_pyfunction!(project_promotion_conflicts, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs new file mode 100644 index 0000000000..40457a7944 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs @@ -0,0 +1,231 @@ +//! Load the project and local configuration files into their Python config objects. + +use std::path::PathBuf; + +use pyo3::exceptions::{PyOSError, PyValueError}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_config::project::errors::{ConfigFailure, FailureClass, LoadFailure}; +use sqlbuild_config::project::main::load_local_config::load_local_config; +use sqlbuild_config::project::main::load_project_config::load_project_config; +use sqlbuild_config::project::main::local_sql_analysis_failure::local_sql_analysis_failure; +use sqlbuild_config::project::main::resolve_config_file::resolve_config_file; +use sqlbuild_config::project::models::{Built, ConfigFile, ConfigPaths}; + +use crate::bindings::_helpers::boundary::config_values::config_value_to_python; + +const MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const PROJECT_CONFIG_ERROR: &str = "ProjectConfigError"; +const FILE_NAMES: [&str; 4] = [ + "sqlbuild_project.toml", + "sqlbuild_project.yml", + "sqlbuild_local.toml", + "sqlbuild_local.yml", +]; +/// The module of each `StrEnum` a loaded config holds. +const ENUM_MODULES: [(&str, &str); 10] = [ + ("ColumnContractMode", "sqlbuild.spec.contracts.types"), + ("FutureCursorAction", "sqlbuild.spec.contracts.types"), + ("MicrobatchLimitAction", "sqlbuild.spec.contracts.types"), + ("TableType", "sqlbuild.spec.contracts.types"), + ("TableTypeDowngradePolicy", "sqlbuild.spec.contracts.types"), + ("RetentionDecreasePolicy", "sqlbuild.spec.contracts.types"), + ( + "MissingMigrationOriginPolicy", + "sqlbuild.spec.contracts.types", + ), + ("EventExportSeverity", "sqlbuild.spec.contracts.types"), + ("AuditSeverity", "sqlbuild.compiler.auditing.types"), + ("CollectionRendering", "sqlbuild.sql_values.types"), +]; + +/// The four configuration file paths of a project directory, as Python and native paths. +struct Files<'py> { + python: Vec>, + native: ConfigPaths, +} + +fn files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { + let python = FILE_NAMES + .iter() + .map(|name| project_dir.call_method1("__truediv__", (*name,))) + .collect::>>()?; + let native: Vec = python + .iter() + .map(|path| path.extract::()) + .collect::>()?; + let [project, legacy_project, local, legacy_local]: [PathBuf; 4] = native + .try_into() + .map_err(|_| PyOSError::new_err("configuration paths"))?; + Ok(Files { + python, + native: ConfigPaths { + project, + legacy_project, + local, + legacy_local, + }, + }) +} + +fn file_index(file: ConfigFile) -> usize { + match file { + ConfigFile::Project => 0, + ConfigFile::LegacyProject => 1, + ConfigFile::Local => 2, + ConfigFile::LegacyLocal => 3, + } +} + +fn config_error(py: Python<'_>, failure: &ConfigFailure, files: &Files<'_>) -> PyResult { + use sqlbuild_config::project::errors::MessagePiece; + let pieces = failure + .message + .iter() + .map(|piece| match piece { + MessagePiece::Text(text) => Ok(PyString::new(py, text).into_any()), + MessagePiece::File(file) => files.python[file_index(*file)].str().map(Bound::into_any), + }) + .collect::>>()?; + let message = PyString::new(py, "").call_method1("join", (pieces,))?; + if failure.class == FailureClass::IntegerDigits { + return Ok(PyValueError::new_err(message.unbind())); + } + let keywords = PyDict::new(py); + keywords.set_item("help", failure.help.as_deref())?; + let error = py + .import(ERRORS_MODULE)? + .getattr(PROJECT_CONFIG_ERROR)? + .call((message,), Some(&keywords))?; + Ok(PyErr::from_value(error)) +} + +fn os_error(py: Python<'_>, error: &std::io::Error, path: &Bound<'_, PyAny>) -> PyResult { + let Some(errno) = error.raw_os_error() else { + return Ok(PyOSError::new_err(error.to_string())); + }; + let reason = py.import("os")?.call_method1("strerror", (errno,))?; + let value = py + .import("builtins")? + .getattr("OSError")? + .call1((errno, reason, path.str()?))?; + Ok(PyErr::from_value(value)) +} + +fn load_error(py: Python<'_>, failure: &LoadFailure, files: &Files<'_>) -> PyResult { + match failure { + LoadFailure::Config(failure) => config_error(py, failure, files), + LoadFailure::Io(file, error) => os_error(py, error, &files.python[file_index(*file)]), + LoadFailure::Utf8(bytes) => Ok(PyBytes::new(py, bytes) + .call_method0("decode") + .err() + .unwrap_or_else(|| PyOSError::new_err("invalid UTF-8"))), + } +} + +fn enum_member<'py>(py: Python<'py>, class: &str, value: &str) -> PyResult> { + let module = ENUM_MODULES + .iter() + .find(|(name, _)| *name == class) + .map_or("sqlbuild.spec.contracts.types", |(_, module)| *module); + py.import(module)?.getattr(class)?.call1((value,)) +} + +fn object<'py>( + py: Python<'py>, + class: &str, + fields: Vec<(&'static str, Built)>, +) -> PyResult> { + let keywords = PyDict::new(py); + for (name, value) in fields { + keywords.set_item(name, built(py, value)?)?; + } + py.import(MODELS_MODULE)? + .getattr(class)? + .call((), Some(&keywords)) +} + +fn built(py: Python<'_>, value: Built) -> PyResult> { + match value { + Built::Value(value) => config_value_to_python(py, value), + Built::Object(class, fields) => Ok(object(py, class, fields)?.unbind()), + Built::Enum(class, member) => Ok(enum_member(py, class, &member)?.unbind()), + Built::Dict(entries) => { + let mapping = PyDict::new(py); + for (key, item) in entries { + mapping.set_item(key, built(py, item)?)?; + } + Ok(mapping.into_any().unbind()) + } + Built::Tuple(items) => PyTuple::new(py, items)?.into_py_any(py), + Built::FrozenSet(items) => PyFrozenSet::new(py, items)?.into_py_any(py), + Built::Decimal(value) => { + let number = config_value_to_python(py, value)?; + let text = number.bind(py).str()?; + Ok(py + .import("decimal")? + .getattr("Decimal")? + .call1((text,))? + .unbind()) + } + } +} + +/// `load_project_config`: the validated `ProjectConfig` of `project_dir`. +#[pyfunction] +fn load_project_config_file(py: Python<'_>, project_dir: &Bound<'_, PyAny>) -> PyResult> { + let files = files(project_dir)?; + match load_project_config(&files.native) { + Ok(config) => built(py, config), + Err(failure) => Err(load_error(py, &failure, &files)?), + } +} + +/// `load_local_config`: the validated `LocalConfig` of `project_dir`, or the default. +#[pyfunction] +fn load_local_config_file(py: Python<'_>, project_dir: &Bound<'_, PyAny>) -> PyResult> { + let files = files(project_dir)?; + match load_local_config(&files.native) { + Ok(config) => built(py, config), + Err(failure) => Err(load_error(py, &failure, &files)?), + } +} + +/// The project or local configuration file path, rejecting an unconverted legacy file. +#[pyfunction] +fn resolve_config_file_path<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, + local: bool, +) -> PyResult> { + let files = files(project_dir)?; + match resolve_config_file(&files.native, local) { + Ok(file) => Ok(files.python[file_index(file)].clone()), + Err(failure) => Err(config_error(py, &failure, &files)?), + } +} + +/// `validate_local_sql_analysis_policy` from the project requirement and the local setting. +#[pyfunction] +fn validate_local_sql_analysis( + py: Python<'_>, + project_dir: &Bound<'_, PyAny>, + flags: (bool, bool, bool), +) -> Result<(), PyErr> { + let files = files(project_dir)?; + match local_sql_analysis_failure(&files.native, flags) { + None => Ok(()), + Some(failure) => Err(config_error(py, &failure, &files)?), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(load_project_config_file, module)?)?; + module.add_function(wrap_pyfunction!(load_local_config_file, module)?)?; + module.add_function(wrap_pyfunction!(resolve_config_file_path, module)?)?; + module.add_function(wrap_pyfunction!(validate_local_sql_analysis, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs index 894c76182c..2f37dc5d0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -2,8 +2,16 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::contracts::model_contracts; +use crate::bindings::_helpers::contracts::{ + dynamic_names, effective_config, model_contracts, project_config, resource_identities, + sql_values, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - model_contracts::register(module) + model_contracts::register(module)?; + dynamic_names::register(module)?; + effective_config::register(module)?; + project_config::register(module)?; + resource_identities::register(module)?; + sql_values::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs new file mode 100644 index 0000000000..b890e7365c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs @@ -0,0 +1,20 @@ +//! Canonical snake_case resource identity checks. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyString, PyStringMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::resource_identities::main::resource_identity_failure::resource_identity_failure; + +/// The snake_case suggestion and help for a non-canonical identity, or `None`. +#[pyfunction] +fn resource_identity_suggestion( + name: &Bound<'_, PyString>, + private_identity: bool, +) -> PyResult> { + let text = name.to_string_lossy(); + Ok(resource_identity_failure(&text, private_identity)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resource_identity_suggestion, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs new file mode 100644 index 0000000000..c289f13901 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/sql_values.rs @@ -0,0 +1,217 @@ +//! Typed SQL value normalization between Python authored values and native `SqlValue`s. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, + PyStringMethods, PyTuple, +}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::models::{SqlLogicalType, SqlValue}; +use sqlbuild_model_config::sql_values::errors::SqlValueFailure; +use sqlbuild_model_config::sql_values::main::normalize_sql_value::normalize_sql_value; +use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue, SqlValueLimits}; + +const MODELS_MODULE: &str = "sqlbuild.sql_values.models"; +const TYPES_MODULE: &str = "sqlbuild.sql_values.types"; +const EXCEPTIONS_MODULE: &str = "sqlbuild.sql_values.exceptions"; + +/// The Python classes authored values are told apart by and normalized values are built from. +pub(crate) struct SqlValueClasses<'py> { + decimal: Bound<'py, PyAny>, + authored_set: Bound<'py, PyAny>, + value_call: Bound<'py, PyAny>, + sql_value: Bound<'py, PyAny>, + logical_type: Bound<'py, PyAny>, + kind: Bound<'py, PyAny>, +} + +impl<'py> SqlValueClasses<'py> { + pub(crate) fn load(py: Python<'py>) -> PyResult { + let models = py.import(MODELS_MODULE)?; + Ok(Self { + decimal: py.import("decimal")?.getattr("Decimal")?, + authored_set: models.getattr("AuthoredSqlSet")?, + value_call: models.getattr("AuthoredSqlValueCall")?, + sql_value: models.getattr("SqlValue")?, + logical_type: models.getattr("SqlLogicalType")?, + kind: py.import(TYPES_MODULE)?.getattr("SqlValueKind")?, + }) + } +} + +fn is_exactly(value: &Bound<'_, PyAny>, class: &Bound<'_, PyAny>) -> bool { + value.get_type().is(class) +} + +fn sequence<'py>( + items: impl Iterator>, + classes: &SqlValueClasses<'_>, +) -> PyResult> { + items.map(|item| raw_value(&item, classes)).collect() +} + +/// An authored Python value read by exact type, as `normalize_sql_value` dispatches. +pub(crate) fn raw_value( + value: &Bound<'_, PyAny>, + classes: &SqlValueClasses<'_>, +) -> PyResult { + let py = value.py(); + if value.is_none() { + return Ok(RawSqlValue::Null); + } + if is_exactly(value, &py.get_type::().into_any()) { + return Ok(RawSqlValue::Bool(value.extract()?)); + } + if is_exactly(value, &py.get_type::().into_any()) { + return Ok(match value.extract::() { + Ok(number) => RawSqlValue::Int(number), + Err(_) => RawSqlValue::BigInt(value.str()?.to_string_lossy().into_owned()), + }); + } + if is_exactly(value, &py.get_type::().into_any()) { + return Ok(RawSqlValue::Float(value.extract()?)); + } + if is_exactly(value, &classes.decimal) { + return Ok(RawSqlValue::Decimal( + value.str()?.to_string_lossy().into_owned(), + )); + } + if is_exactly(value, &py.get_type::().into_any()) { + return Ok(RawSqlValue::Str( + value.downcast::()?.to_string_lossy().into_owned(), + )); + } + if is_exactly(value, &py.get_type::().into_any()) { + return sequence(value.downcast::()?.iter(), classes).map(RawSqlValue::List); + } + if value.is_instance(&classes.authored_set)? { + let items = value.getattr("values")?; + return sequence( + items.try_iter()?.collect::>>()?.into_iter(), + classes, + ) + .map(RawSqlValue::Set); + } + if value.is_instance(&classes.value_call)? { + let mut arguments: Vec<(String, RawSqlValue)> = Vec::new(); + for argument in value.getattr("arguments")?.try_iter()? { + let (name, item): (String, Bound<'_, PyAny>) = argument?.extract()?; + arguments.push((name, raw_value(&item, classes)?)); + } + return Ok(RawSqlValue::Call(arguments)); + } + if is_exactly(value, &py.get_type::().into_any()) { + let mut entries: Vec<(RawKey, RawSqlValue)> = Vec::new(); + for (key, item) in value.downcast::()?.iter() { + let raw_key = if is_exactly(&key, &py.get_type::().into_any()) { + RawKey::Str(key.downcast::()?.to_string_lossy().into_owned()) + } else { + RawKey::Other(key.repr()?.to_string_lossy().into_owned()) + }; + entries.push((raw_key, raw_value(&item, classes)?)); + } + return Ok(RawSqlValue::Object(entries)); + } + let name = value.get_type().getattr("__name__")?; + Ok(RawSqlValue::Other( + name.str()?.to_string_lossy().into_owned(), + )) +} + +fn logical_type<'py>( + logical: &SqlLogicalType, + classes: &SqlValueClasses<'py>, +) -> PyResult> { + let kind = classes.kind.call1((logical.kind.as_str(),))?; + let element = match &logical.element_type { + Some(element) => logical_type(element, classes)?, + None => classes.kind.py().None().into_bound(classes.kind.py()), + }; + classes.logical_type.call1((kind, element)) +} + +fn payload<'py>(value: &SqlValue, classes: &SqlValueClasses<'py>) -> PyResult> { + let py = classes.kind.py(); + Ok(match value { + SqlValue::Null => py.None().into_bound(py), + SqlValue::Boolean(flag) => flag.into_bound_py_any(py)?, + SqlValue::Integer(number) => number.into_bound_py_any(py)?, + SqlValue::Float(number) => number.into_bound_py_any(py)?, + SqlValue::Decimal(text) => classes.decimal.call1((text,))?, + SqlValue::String(text) => text.into_bound_py_any(py)?, + SqlValue::List { values, .. } | SqlValue::Set { values, .. } => PyTuple::new( + py, + values + .iter() + .map(|item| sql_value(item, classes)) + .collect::>>()?, + )? + .into_any(), + SqlValue::Object(entries) => PyTuple::new( + py, + entries + .iter() + .map(|(key, item)| { + PyTuple::new(py, [key.into_bound_py_any(py)?, sql_value(item, classes)?]) + }) + .collect::>>()?, + )? + .into_any(), + }) +} + +/// A Python `SqlValue` for a native one. +pub(crate) fn sql_value<'py>( + value: &SqlValue, + classes: &SqlValueClasses<'py>, +) -> PyResult> { + classes.sql_value.call1(( + logical_type(&value.logical_type(), classes)?, + payload(value, classes)?, + )) +} + +/// The exception a normalization failure raises. +pub(crate) fn sql_value_error(py: Python<'_>, failure: SqlValueFailure) -> PyResult { + Ok(match failure { + SqlValueFailure::Invalid(message) => PyErr::from_value( + py.import(EXCEPTIONS_MODULE)? + .getattr("SqlValueValidationError")? + .call1((message,))?, + ), + SqlValueFailure::DecimalOverflow => { + let overflow = py.import("decimal")?.getattr("Overflow")?; + let classes = pyo3::types::PyList::new(py, [overflow.clone()])?; + PyErr::from_value(overflow.call1((classes,))?) + } + }) +} + +/// `normalize_sql_value`: the canonical `SqlValue` of an authored value. +#[pyfunction] +fn normalize_typed_sql_value<'py>( + py: Python<'py>, + raw_value: &Bound<'py, PyAny>, + typing: (String, Option), + limits: (i64, i64, i64), +) -> PyResult> { + let (context, explicit_type) = typing; + let classes = SqlValueClasses::load(py)?; + let raw = self::raw_value(raw_value, &classes)?; + let (max_depth, max_elements, max_size) = limits; + let limits = SqlValueLimits { + max_depth, + max_elements, + max_size, + }; + match normalize_sql_value(&raw, &context, explicit_type.as_deref(), limits) { + Ok(value) => sql_value(&value, &classes), + Err(failure) => Err(sql_value_error(py, failure)?), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(normalize_typed_sql_value, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/globbing.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/globbing.rs new file mode 100644 index 0000000000..7366a9f09a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/globbing.rs @@ -0,0 +1,39 @@ +//! `Path.glob` name matching for the discovery directory snapshot. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyAnyMethods, PyString, PyStringMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::text::main::python_glob_name_match::python_glob_name_match; + +const ENCODE: &str = "encode"; +const UTF32_LE: &str = "utf-32-le"; +const SURROGATE_PASS: &str = "surrogatepass"; + +/// The code points of `name`; undecodable file names keep their lone surrogates. +fn code_points(name: &Bound<'_, PyString>) -> PyResult> { + match name.to_str() { + Ok(text) => Ok(text.chars().map(u32::from).collect()), + Err(_) => Ok(name + .call_method1(ENCODE, (UTF32_LE, SURROGATE_PASS))? + .extract::>()? + .as_chunks::<4>() + .0 + .iter() + .map(|unit| u32::from_le_bytes([unit[0], unit[1], unit[2], unit[3]])) + .collect()), + } +} + +/// Whether a directory entry name matches a single-name glob `pattern`. +#[pyfunction] +fn glob_name_match(name: &Bound<'_, PyString>, pattern: &str, ignore_case: bool) -> PyResult { + Ok(python_glob_name_match( + &code_points(name)?, + pattern, + ignore_case, + )) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(glob_name_match, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs new file mode 100644 index 0000000000..06d34a6a1d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs @@ -0,0 +1,198 @@ +//! Model-header enums and constants, and typed constant file entries, validated natively. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::declarations::errors::DeclarationFailure; +use sqlbuild_model_config::declarations::main::model_constant_declarations::model_constant_declarations; +use sqlbuild_model_config::declarations::main::model_enum_declarations::model_enum_declarations; +use sqlbuild_model_config::declarations::main::typed_constant_declaration::typed_constant_declaration; +use sqlbuild_model_config::declarations::models::{ + ConstantFacts, DeclarationSite, EnumFacts, EnumValue, +}; +use sqlbuild_model_config::sql_values::errors::SqlValueFailure; +use sqlbuild_model_config::sql_values::models::RawSqlValue; + +use crate::bindings::_helpers::contracts::sql_values::{ + SqlValueClasses, raw_value, sql_value, sql_value_error, +}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const IDENTITY_ERRORS_MODULE: &str = "sqlbuild.compiler.resource_names.exceptions"; +const RENDERING_MODULE: &str = "sqlbuild.sql_values.types"; + +fn declaration_error(py: Python<'_>, failure: DeclarationFailure) -> PyResult { + Ok(match failure { + DeclarationFailure::Parse(message) => PyErr::from_value( + py.import(DISCOVERY_ERRORS_MODULE)? + .getattr("DeclarationParseError")? + .call1((message,))?, + ), + DeclarationFailure::Identity(message, help) => { + let keywords = PyDict::new(py); + keywords.set_item("help", help)?; + PyErr::from_value( + py.import(IDENTITY_ERRORS_MODULE)? + .getattr("ResourceIdentityError")? + .call((message,), Some(&keywords))?, + ) + } + DeclarationFailure::DecimalOverflow => { + sql_value_error(py, SqlValueFailure::DecimalOverflow)? + } + }) +} + +/// Who owns built declarations: their relative path and model. +struct Owner<'py> { + relative_path: Bound<'py, PyAny>, + model_name: Option, +} + +fn enum_record<'py>( + py: Python<'py>, + facts: EnumFacts, + owner: &Owner<'py>, +) -> PyResult> { + let models = py.import(MODELS_MODULE)?; + let member_class = models.getattr("EnumMember")?; + let int_class = py.import("builtins")?.getattr("int")?; + let members = facts + .members + .into_iter() + .map(|(name, value)| { + let value = match value { + EnumValue::Text(text) => text.into_bound_py_any(py)?, + EnumValue::Integer(digits) => int_class.call1((digits,))?, + }; + member_class.call1((name, value)) + }) + .collect::>>()?; + models.getattr("EnumDeclaration")?.call1(( + facts.name, + PyTuple::new(py, members)?, + facts.scalar_type, + &owner.relative_path, + owner.model_name.as_deref(), + )) +} + +fn constant_record<'py>( + py: Python<'py>, + facts: ConstantFacts, + owner: &Owner<'py>, + classes: &SqlValueClasses<'py>, +) -> PyResult> { + let render_as = match facts.render_as { + Some(rendering) => py + .import(RENDERING_MODULE)? + .getattr("CollectionRendering")? + .call1((rendering,))?, + None => py.None().into_bound(py), + }; + py.import(MODELS_MODULE)? + .getattr("ConstantDeclaration")? + .call1(( + facts.name, + sql_value(&facts.value, classes)?, + &owner.relative_path, + owner.model_name.as_deref(), + render_as, + )) +} + +fn optional_raw( + value: Option<&Bound<'_, PyAny>>, + classes: &SqlValueClasses<'_>, +) -> PyResult> { + value + .filter(|value| !value.is_none()) + .map(|value| raw_value(value, classes)) + .transpose() +} + +type HeaderDeclarations<'py> = (Bound<'py, PyTuple>, Bound<'py, PyTuple>); + +/// `parse_model_enum_declarations` then `parse_model_constant_declarations` for one model. +#[pyfunction] +fn model_header_declarations<'py>( + py: Python<'py>, + header: (Option>, Option>), + model_name: String, + relative_path: Bound<'py, PyAny>, +) -> PyResult> { + let (enums, constants) = header; + let classes = SqlValueClasses::load(py)?; + let file_path = relative_path.str()?.to_string(); + let site = DeclarationSite { + file_path: &file_path, + model_name: Some(&model_name), + }; + let owner = Owner { + relative_path: relative_path.clone(), + model_name: Some(model_name.clone()), + }; + let raw_enums = optional_raw(enums.as_ref(), &classes)?; + let enum_facts = match model_enum_declarations(raw_enums.as_ref(), site) { + Ok(facts) => facts, + Err(failure) => return Err(declaration_error(py, failure)?), + }; + let raw_constants = optional_raw(constants.as_ref(), &classes)?; + let constant_facts = match model_constant_declarations(raw_constants.as_ref(), site) { + Ok(facts) => facts, + Err(failure) => return Err(declaration_error(py, failure)?), + }; + let enum_records = enum_facts + .into_iter() + .map(|facts| enum_record(py, facts, &owner)) + .collect::>>()?; + let constant_records = constant_facts + .into_iter() + .map(|facts| constant_record(py, facts, &owner, &classes)) + .collect::>>()?; + Ok(( + PyTuple::new(py, enum_records)?, + PyTuple::new(py, constant_records)?, + )) +} + +/// `typed_constant_declaration` for a public constant file entry. +#[pyfunction] +fn typed_constant_record<'py>( + py: Python<'py>, + constant: (String, Bound<'py, PyAny>), + options: (Option, Option), + paths: (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let (name, value) = constant; + let (explicit_type, render_as) = options; + let (file_path, relative_path) = paths; + let classes = SqlValueClasses::load(py)?; + let raw = raw_value(&value, &classes)?; + let path = file_path.str()?.to_string(); + match typed_constant_declaration( + &name, + &raw, + (explicit_type.as_deref(), render_as.as_deref()), + &path, + ) { + Ok(facts) => constant_record( + py, + facts, + &Owner { + relative_path, + model_name: None, + }, + &classes, + ), + Err(failure) => Err(declaration_error(py, failure)?), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(model_header_declarations, module)?)?; + module.add_function(wrap_pyfunction!(typed_constant_record, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/key_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/key_checks.rs new file mode 100644 index 0000000000..a19f1a9e74 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/key_checks.rs @@ -0,0 +1,27 @@ +//! Unknown and unsupported authored keys, rejected with nearest-key suggestions natively. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::key_checks::main::unknown_mapping_keys::unknown_mapping_keys as unknown_keys; +use sqlbuild_discovery::key_checks::main::unsupported_keys_help::unsupported_keys_help as keys_help; + +/// `(message after the label, help)` when a mapping key is outside `allowed`. +#[pyfunction] +fn unknown_mapping_keys( + keys: Vec<(String, bool)>, + allowed: Vec, +) -> Option<(String, String)> { + unknown_keys(&keys, &allowed) +} + +/// `unsupported_keys_help` for `keys` outside `supported_keys`. +#[pyfunction] +fn unsupported_keys_help(keys: Vec, supported_keys: Vec) -> String { + let keys: Vec<&str> = keys.iter().map(String::as_str).collect(); + keys_help(&keys, &supported_keys) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(unknown_mapping_keys, module)?)?; + module.add_function(wrap_pyfunction!(unsupported_keys_help, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs new file mode 100644 index 0000000000..d01171a01d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs @@ -0,0 +1,37 @@ +//! Authored locations Python consumers read: YAML entry lines and SQL-test helper spans. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::sql_test_attachment::main::sql_test_cte_location::sql_test_cte_location; +use sqlbuild_analysis::sql_test_attachment::models::CteLocation; +use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; + +/// Each outermost `- name:` entry name with its 1-based line. +#[pyfunction] +fn yaml_entry_line_index(contents: &str) -> Vec<(String, usize)> { + yaml_entry_lines(contents) +} + +/// `(line, column, end line, end column)` of a helper problem in one SQL test block. +#[pyfunction] +#[pyo3(signature = (contents, sql_body, cte_name, call))] +fn sql_test_cte_span( + contents: &str, + sql_body: &str, + cte_name: &str, + call: Option<&str>, +) -> (usize, usize, usize, usize) { + let location: CteLocation = sql_test_cte_location(contents, sql_body, cte_name, call); + ( + location.line, + location.column, + location.end_line, + location.end_column, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(yaml_entry_line_index, module)?)?; + module.add_function(wrap_pyfunction!(sql_test_cte_span, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 37c0084a2e..40244e1477 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -1,3 +1,8 @@ pub(crate) mod declaration_files; +pub(crate) mod globbing; +pub(crate) mod header_declarations; +pub(crate) mod key_checks; +pub(crate) mod locations; pub(crate) mod project_files; pub(crate) mod registration; +pub(crate) mod sql_test_blocks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index 557b064ebe..d4c6034ae8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -2,10 +2,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::discovery::{declaration_files, project_files}; +use crate::bindings::_helpers::discovery::{ + declaration_files, globbing, header_declarations, key_checks, locations, project_files, + sql_test_blocks, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { project_files::register(module)?; declaration_files::register(module)?; + locations::register(module)?; + sql_test_blocks::register(module)?; + header_declarations::register(module)?; + key_checks::register(module)?; + globbing::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs new file mode 100644 index 0000000000..6a128ec24b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -0,0 +1,138 @@ +//! SQL `TEST(...)` blocks validated natively and built as discovery records. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::sql_tests::main::sql_test_block::sql_test_block; +use sqlbuild_model_config::sql_tests::main::sql_test_names_failure::sql_test_names_failure; +use sqlbuild_model_config::sql_tests::models::{SqlTestBlock, SqlTestCase}; +use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue}; + +use crate::bindings::_helpers::contracts::sql_values::{SqlValueClasses, raw_value, sql_value}; + +const MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const SQL_VALUE_TYPES_MODULE: &str = "sqlbuild.sql_values.types"; + +type NativeBlock<'py> = (Bound<'py, PyDict>, String, Bound<'py, PyAny>); + +fn parse_error(py: Python<'_>, message: String) -> PyResult { + Ok(PyErr::from_value( + py.import(ERRORS_MODULE)? + .getattr("SqlTestParseError")? + .call1((message,))?, + )) +} + +fn header_entries( + header: &Bound<'_, PyDict>, + classes: &SqlValueClasses<'_>, +) -> PyResult> { + match raw_value(header.as_any(), classes)? { + RawSqlValue::Object(entries) => Ok(entries), + _ => Ok(Vec::new()), + } +} + +fn case_record<'py>( + py: Python<'py>, + (case_index, case): (usize, &SqlTestCase), + classes: &SqlValueClasses<'py>, +) -> PyResult> { + let mut values: Vec> = Vec::with_capacity(case.values.len()); + for (name, value) in &case.values { + values.push(PyTuple::new( + py, + [name.into_bound_py_any(py)?, sql_value(value, classes)?], + )?); + } + py.import(MODELS_MODULE)? + .getattr("DiscoveredSqlTestCase")? + .call1((case.name.as_str(), PyTuple::new(py, values)?, case_index)) +} + +fn block_record<'py>( + py: Python<'py>, + (index, native): (usize, &NativeBlock<'py>), + block: SqlTestBlock, + classes: &SqlValueClasses<'py>, +) -> PyResult> { + let models = py.import(MODELS_MODULE)?; + let kind = py.import(SQL_VALUE_TYPES_MODULE)?.getattr("SqlValueKind")?; + let parameters = block + .parameters + .iter() + .map(|parameter| { + models.getattr("SqlTestParameterDeclaration")?.call1(( + parameter.name.as_str(), + kind.call1((parameter.value_type,))?, + parameter.nullable, + )) + }) + .collect::>>()?; + let cases = block + .cases + .iter() + .enumerate() + .map(|(case_index, case)| case_record(py, (case_index, case), classes)) + .collect::>>()?; + let keywords = PyDict::new(py); + keywords.set_item("test_index", index)?; + keywords.set_item("header_values", &native.0)?; + keywords.set_item("sql_body", native.1.as_str())?; + keywords.set_item("name", block.name)?; + keywords.set_item( + "mode", + py.import(TYPES_MODULE)? + .getattr("SqlTestMode")? + .call1((block.mode,))?, + )?; + keywords.set_item("parameters", PyTuple::new(py, parameters)?)?; + keywords.set_item("cases", PyTuple::new(py, cases)?)?; + keywords.set_item("cursor_start", block.cursor_start)?; + keywords.set_item("cursor_end", block.cursor_end)?; + keywords.set_item("sql_body_span", &native.2)?; + models + .getattr("DiscoveredSqlTestBlock")? + .call((), Some(&keywords)) +} + +/// Each `(header, sql_body, span)` block of a SQL test file as a `DiscoveredSqlTestBlock`. +#[pyfunction] +fn sql_test_blocks<'py>( + py: Python<'py>, + file_path: &str, + blocks: Vec>, +) -> PyResult>> { + let classes = SqlValueClasses::load(py)?; + let mut records: Vec> = Vec::with_capacity(blocks.len()); + for (offset, native) in blocks.iter().enumerate() { + let header = header_entries(&native.0, &classes)?; + match sql_test_block(&header, &native.1, file_path) { + Ok(block) => records.push(block_record(py, (offset + 1, native), block, &classes)?), + Err(message) => return Err(parse_error(py, message)?), + } + } + Ok(records) +} + +/// `validate_sql_test_names` over each block's `(test_index, name)`. +#[pyfunction] +fn validate_sql_test_block_names( + py: Python<'_>, + file_path: &str, + blocks: Vec<(usize, Option)>, +) -> Result<(), PyErr> { + match sql_test_names_failure(file_path, &blocks) { + None => Ok(()), + Some(message) => Err(parse_error(py, message)?), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(sql_test_blocks, module)?)?; + module.add_function(wrap_pyfunction!(validate_sql_test_block_names, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs new file mode 100644 index 0000000000..b828b7333b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod project_graph; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs new file mode 100644 index 0000000000..b3783d0db2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -0,0 +1,260 @@ +//! The compiled project graph and selector resolution, held natively for Python's graph users. + +use std::collections::HashMap; + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; +use sqlbuild_analysis::graph::main::build_resources::build_resources; +use sqlbuild_analysis::graph::main::edge_keys::edge_keys; +use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; +use sqlbuild_analysis::graph::main::match_selector::match_selector; +use sqlbuild_analysis::graph::main::model_layer_count::model_layer_count; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::main::resolve_selector_tokens::resolve_selector_tokens; +use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; +use sqlbuild_analysis::graph::main::selector_name_help::selector_name_help; +use sqlbuild_analysis::graph::main::unique_id_edges::unique_id_edges; +use sqlbuild_analysis::graph::models::{ + BuildResources, EdgeDirection, GraphIndexes, GraphResource, ParsedSelector, ProjectGraph, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +const KIND_SELECTOR: &str = "kind"; +const PATH_SELECTOR: &str = "path"; + +/// `(key, deps, tags, model folder)`. +pub(crate) type ResourceRow = (ObjectKey, Vec, Vec, Option); +type Edges = Vec<(ObjectKey, Vec)>; +/// `(names, upstream, downstream, tags, paths)` in Python's dict order. +type IndexesRow = ( + Vec<(String, ObjectKey)>, + Edges, + Edges, + Vec<(String, Vec)>, + Vec<(ObjectKey, String)>, +); +/// `(unique id, neighbour unique ids)` in graph order. +type UniqueIdMap = Vec<(String, Vec)>; +/// `PlannerInputError`'s `(code, message, help)`. +type FailureRow = (&'static str, String, Option); +/// The keys, or the `PlannerInputError` Python raises instead. +type KeysOutcome = (Option>, Option); +/// `("kind", kind, value, upstream, downstream)` or `("path", start, end, upstream, downstream)`. +type ParsedRow = (&'static str, String, String, bool, bool); + +/// A project graph: lineage edges, tag, path and name indexes, and selector resolution. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeProjectGraph { + graph: ProjectGraph, +} + +fn dependency_edges(key: &ObjectKey, deps: &[ObjectKey]) -> Vec<(ObjectKey, ObjectKey)> { + deps.iter().map(|dep| (dep.clone(), key.clone())).collect() +} + +fn outcome(result: Result, SelectorError>) -> KeysOutcome { + match result { + Ok(keys) => (Some(keys), None), + Err(error) => (None, Some(failure(error))), + } +} + +fn failure(error: SelectorError) -> FailureRow { + (error.code, error.message, error.help) +} + +impl NativeProjectGraph { + /// Wrap a graph another native stage built, such as project assembly. + pub(crate) fn new(graph: ProjectGraph) -> Self { + Self { graph } + } +} + +/// The project graph indexes of these resources, in project order. +pub(crate) fn project_graph(resources: Vec) -> PyResult { + compiler_guard(|| { + let resources: Vec = resources + .into_iter() + .map(|(key, deps, tags, folder)| GraphResource { + key, + deps, + tags, + folder, + }) + .collect(); + Ok(NativeProjectGraph::new(build_project_graph(&resources))) + }) +} + +#[pymethods] +impl NativeProjectGraph { + /// Build the indexes from resources in project order: models, sources, seeds, functions. + #[staticmethod] + fn from_resources(resources: Vec) -> PyResult { + project_graph(resources) + } + + /// Wrap indexes a caller already built, such as the planner's. + #[staticmethod] + fn from_indexes(indexes: IndexesRow) -> PyResult { + compiler_guard(|| { + let (names, upstream, downstream, tags, paths) = indexes; + Ok(Self::new(graph_from_indexes(GraphIndexes { + names, + upstream, + downstream, + tags, + paths, + }))) + }) + } + + /// `(names, upstream, downstream, tags, paths)` for Python's `ProjectGraph`. + fn indexes(&self) -> IndexesRow { + let graph: &ProjectGraph = &self.graph; + ( + graph.names.clone(), + graph.upstream.clone(), + graph.downstream.clone(), + graph.tags.clone(), + graph.paths.clone(), + ) + } + + /// The key's direct upstream lineage keys, in authored dependency order. + fn upstream(&self, key: ObjectKey) -> Vec { + edge_keys(&self.graph, &key, EdgeDirection::Upstream).to_vec() + } + + /// The key's direct downstream keys, sorted by `(resource type, name)`. + fn downstream(&self, key: ObjectKey) -> Vec { + edge_keys(&self.graph, &key, EdgeDirection::Downstream).to_vec() + } + + /// Every selectable key, in project order with later same-name resources replacing earlier. + fn keys(&self) -> Vec { + self.graph + .names + .iter() + .map(|(_, key)| key.clone()) + .collect() + } + + /// `(selector name, key)` in project order. + fn names(&self) -> Vec<(String, ObjectKey)> { + self.graph.names.clone() + } + + /// Every lineage edge as `(dependency, dependent)`, in upstream order. + fn edges(&self) -> Vec<(ObjectKey, ObjectKey)> { + self.graph + .upstream + .iter() + .flat_map(|(key, deps)| dependency_edges(key, deps)) + .collect() + } + + /// The manifest's `(parent_map, child_map)` entries as dbt-style unique ids, in graph order. + fn unique_id_maps( + &self, + project_name: &str, + prefixes: HashMap, + ) -> (UniqueIdMap, UniqueIdMap) { + ( + unique_id_edges( + &self.graph, + EdgeDirection::Upstream, + project_name, + &prefixes, + ), + unique_id_edges( + &self.graph, + EdgeDirection::Downstream, + project_name, + &prefixes, + ), + ) + } + + /// The compile report's execution layer count over models. + fn model_layer_count(&self) -> usize { + model_layer_count(&self.graph) + } + + /// `--select` minus `--exclude`, plus the functions they need to build. + fn resolve(&self, select: Vec, exclude: Vec) -> PyResult { + compiler_guard(|| Ok(outcome(resolve_selectors(&self.graph, &select, &exclude)))) + } + + /// The keys selector tokens match, without build resources. + fn tokens(&self, selectors: Vec) -> PyResult { + compiler_guard(|| Ok(outcome(resolve_selector_tokens(&self.graph, &selectors)))) + } + + /// The keys one `kind:value` selector names before `+` expansion. + fn matched(&self, kind: &str, value: &str) -> PyResult { + compiler_guard(|| Ok(outcome(match_selector(&self.graph, kind, value)))) + } + + /// The selected keys plus the functions and seeds Python adds to build them. + fn build_resources( + &self, + selected: Vec, + include: (bool, bool, bool), + ) -> PyResult> { + compiler_guard(|| { + let (upstream_functions, upstream_seeds, downstream_functions) = include; + Ok(build_resources( + &self.graph, + &selected, + BuildResources { + upstream_functions, + upstream_seeds, + downstream_functions, + }, + )) + }) + } +} + +/// One selector token parsed, or the `PlannerInputError` Python raises. +#[pyfunction] +fn parse_project_selector(raw: &str) -> (Option, Option) { + match parse_selector(raw) { + Ok(ParsedSelector::Kind { + kind, + value, + upstream, + downstream, + }) => ( + Some((KIND_SELECTOR, kind, value, upstream, downstream)), + None, + ), + Ok(ParsedSelector::Path { + start, + end, + upstream, + downstream, + }) => ( + Some((PATH_SELECTOR, start, end, upstream, downstream)), + None, + ), + Err(error) => (None, Some(failure(error))), + } +} + +/// The "did you mean" help for an unknown selector name, if any. +#[pyfunction] +fn project_selector_name_help(value: &str, candidates: Vec) -> Option { + selector_name_help(value, candidates.iter().map(String::as_str).collect()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(parse_project_selector, module)?)?; + module.add_function(wrap_pyfunction!(project_selector_name_help, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs new file mode 100644 index 0000000000..e0a0e52b9c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/registration.rs @@ -0,0 +1,5 @@ +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + crate::bindings::_helpers::graph::project_graph::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs index 768d0f4744..f0aec5490f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -1,2 +1,4 @@ pub(crate) mod fast_lineage; pub(crate) mod registration; +pub(crate) mod relation_fingerprint; +pub(crate) mod rich_lineage; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs index e22e2d12ee..806db27529 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs @@ -2,8 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::lineage::fast_lineage; +use crate::bindings::_helpers::lineage::{fast_lineage, relation_fingerprint, rich_lineage}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - fast_lineage::register(module) + fast_lineage::register(module)?; + relation_fingerprint::register(module)?; + rich_lineage::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/relation_fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/relation_fingerprint.rs new file mode 100644 index 0000000000..275c8663ea --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/relation_fingerprint.rs @@ -0,0 +1,39 @@ +//! The relation lineage cache key. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::lineage::main::relation_fingerprint::relation_fingerprint; +use sqlbuild_analysis::lineage::models::InterruptedListingPolicy; + +use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; +use crate::bindings::types::CompilerDetach; + +/// The authored files' digest, or `None`; the flag is true where `rglob` raises mid-listing. +#[pyfunction] +fn relation_lineage_fingerprint( + py: Python<'_>, + project_dir: PathBuf, + prefix: Vec, + interrupted_listing_uncacheable: bool, +) -> PyResult> { + let policy = if interrupted_listing_uncacheable { + InterruptedListingPolicy::Uncacheable + } else { + InterruptedListingPolicy::Skip + }; + let fingerprint = py + .compiler_detach(|| Ok(relation_fingerprint(&project_dir, &prefix, policy))) + .map_err(compiler_error)?; + fingerprint.into_digest().map_err(|message| { + NativeCompilerError::new_err(format!( + "NativeCompilerError: relation lineage fingerprint failed: {message}" + )) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(relation_lineage_fingerprint, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs new file mode 100644 index 0000000000..25ba2664d3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/rich_lineage.rs @@ -0,0 +1,160 @@ +//! Rich column lineage for the preview compiler engine. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::lineage::main::build_rich_lineage::build_rich_lineage; +use sqlbuild_analysis::lineage::models::{ + LineageResourceType, LineageSource, RichLineageColumn, RichLineageOutcome, RichLineageRequest, + RichSchemaResource, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::types::CompilerDetach; + +type TypedColumns = Vec<(String, Option)>; +type SourceRow = (&'static str, String, String); +type ColumnRow = ( + String, + &'static str, + &'static str, + &'static str, + Vec, +); +/// `(dialect, [(resource_type, name, assigned, defaulted)], [query_sql])`. +type RichRequestInput = ( + String, + Vec<(String, String, TypedColumns, TypedColumns)>, + Vec, +); +/// `(status, columns, has_star, detail)`: detail is the polyglot error or the deferral kind. +type OutcomeRow = (&'static str, Vec, bool, Option); + +/// Build rich lineage for the requested models' SQL, in request order. +#[pyfunction] +fn build_rich_column_lineage( + py: Python<'_>, + request: RichRequestInput, +) -> PyResult> { + let (dialect, schema, models) = request; + let schema: Vec = schema + .into_iter() + .map(schema_resource) + .collect::>()?; + let request = RichLineageRequest { + dialect, + schema, + models, + }; + let outcomes: Vec = py + .compiler_detach(|| build_rich_lineage(&request)) + .map_err(compiler_error)?; + Ok(outcomes.into_iter().map(outcome_row).collect()) +} + +fn schema_resource( + (resource_type, name, assigned, defaulted): (String, String, TypedColumns, TypedColumns), +) -> PyResult { + let Some(kind) = LineageResourceType::from_value(&resource_type) else { + return Err(PyValueError::new_err(format!( + "unknown resource: {resource_type}" + ))); + }; + Ok(RichSchemaResource { + resource_type: kind, + name, + assigned, + defaulted, + }) +} + +fn outcome_row(outcome: RichLineageOutcome) -> OutcomeRow { + let (status, columns, has_star, detail) = outcome.into_parts(); + ( + status, + columns.into_iter().map(column_row).collect(), + has_star, + detail, + ) +} + +fn column_row(column: RichLineageColumn) -> ColumnRow { + let RichLineageColumn { + column, + nullability, + } = column; + ( + column.output_column, + column.transform_kind.as_str(), + column.confidence.as_str(), + nullability.as_str(), + column + .upstream_columns + .into_iter() + .map(source_row) + .collect(), + ) +} + +fn source_row(source: LineageSource) -> SourceRow { + ( + source.resource_type.as_str(), + source.resource_name, + source.column_name, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_rich_column_lineage, module)?)?; + module.add_function(wrap_pyfunction!(build_direct_semantic_uses, module)?)?; + Ok(()) +} + +type SemanticUseRow = ( + String, + String, + String, + &'static str, + String, + String, + String, + Option, + Option, +); + +/// Project semantic uses onto authored resources without Python analysis reconstruction. +#[pyfunction] +fn build_direct_semantic_uses( + py: Python<'_>, + schema: Vec<(String, String, TypedColumns, TypedColumns)>, + models: Vec<(String, String, String)>, + dialect: &str, +) -> PyResult> { + let schema = schema + .into_iter() + .map(schema_resource) + .collect::>>()?; + let uses = py + .compiler_detach(|| { + sqlbuild_analysis::lineage::main::direct_semantic_uses::direct_semantic_uses( + &schema, &models, dialect, + ) + }) + .map_err(compiler_error)?; + Ok(uses + .into_iter() + .map(|usage| { + ( + usage.model, + usage.context, + usage.expression, + usage.resource_type.as_str(), + usage.resource_name, + usage.column, + usage.confidence, + usage.line, + usage.offset, + ) + }) + .collect()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 148a78e9a8..29b805a100 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -3,11 +3,15 @@ pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod boundary; pub(crate) mod cache; +pub(crate) mod compile_outputs; +pub(crate) mod compiled_project; pub(crate) mod contracts; pub(crate) mod discovery; +pub(crate) mod graph; pub(crate) mod lineage; pub(crate) mod model_config; pub(crate) mod project_assembly; +pub(crate) mod refactor; pub(crate) mod render; pub(crate) mod rules; pub(crate) mod scopes; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/column_migrations.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/column_migrations.rs new file mode 100644 index 0000000000..d7c9eb867a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/column_migrations.rs @@ -0,0 +1,64 @@ +//! Column renames validated natively, raised by Python as the attachment raises them. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::model_validation::main::validate_column_migrations::validate_column_migrations; +use sqlbuild_model_config::model_validation::models::ColumnMigration; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::model_config::config_errors::native_config_error; + +const MATERIALIZED_KEY: &str = "materialized"; +const MIGRATE_FROM_KEY: &str = "migrate_from"; + +/// The first column-rename problem for a model with these config values, if any. +pub(crate) fn column_migration_error( + model_name: &str, + values: &Bound<'_, PyDict>, + columns: &[ColumnMigration], +) -> PyResult> { + if columns.iter().all(|column| column.migrate_from.is_none()) { + return Ok(None); + } + let materialized: Option = match values.get_item(MATERIALIZED_KEY)? { + None => None, + Some(value) if value.is_none() => None, + Some(value) => match value.downcast::() { + Ok(text) => Some(text.to_str()?.to_owned()), + Err(_) => return Ok(Some(ConfigError::config_value_type(MATERIALIZED_KEY))), + }, + }; + let model_migrates: bool = values + .get_item(MIGRATE_FROM_KEY)? + .is_some_and(|value| !value.is_none()); + Ok( + validate_column_migrations(model_name, materialized.as_deref(), model_migrates, columns) + .err(), + ) +} + +/// `None`, or the native error for the model's first column-rename problem. +#[pyfunction] +fn column_migration_problem<'py>( + py: Python<'py>, + model_name: &str, + values: Bound<'py, PyDict>, + columns: Vec<(String, Option)>, +) -> PyResult> { + let columns: Vec = columns + .into_iter() + .map(|(name, migrate_from)| ColumnMigration { name, migrate_from }) + .collect(); + compiler_guard( + || match column_migration_error(model_name, &values, &columns)? { + Some(error) => Ok(native_config_error(py, error)?.into_bound(py).into_any()), + None => Ok(py.None().into_pyobject(py)?.into_any()), + }, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(column_migration_problem, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs index 54a388c461..4f7fe686f6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -28,6 +28,12 @@ const EFFECTIVE_VARS_LABEL: &str = "effective vars"; const PROJECT_VAR_VALUES_MODULE: &str = "sqlbuild.compiler.authored_values.main._project_var_values"; const RENDER_PROJECT_VAR_TEXT: &str = "render_project_var_text"; +const VALUE_PATH: &str = "its value"; +const BUILTINS_MODULE: &str = "builtins"; +const ASCII_FUNCTION: &str = "ascii"; +const ENCODE: &str = "encode"; +const UTF32_LE: &str = "utf-32-le"; +const SURROGATE_PASS: &str = "surrogatepass"; /// Variables, the process environment and context values, all read through Python. pub(crate) struct PythonHost<'py> { @@ -283,6 +289,66 @@ fn expand_config_templates<'py>( outcome(py, host, result, &label) } +/// The first `(var name, value path, UTF-8 byte offset)` of a lone surrogate in raw vars. +#[pyfunction] +fn lone_surrogate_site( + raw_values: Bound<'_, PyDict>, +) -> PyResult, String, usize)>> { + for (key, value) in raw_values.iter() { + if let Some((path, offset)) = surrogate_path(&value, VALUE_PATH.to_owned())? { + return Ok(Some((key.str()?, path, offset))); + } + } + Ok(None) +} + +/// Python's former `_reject_lone_surrogates` walk: dict keys then items, lists and tuples. +fn surrogate_path(value: &Bound<'_, PyAny>, path: String) -> PyResult> { + if let Ok(mapping) = value.downcast::() { + for (key, item) in mapping.iter() { + if let Some(site) = surrogate_path(&key, format!("{path} key"))? { + return Ok(Some(site)); + } + let key_ascii = PyModule::import(value.py(), BUILTINS_MODULE)? + .getattr(ASCII_FUNCTION)? + .call1((&key,))? + .str()? + .to_string_lossy() + .into_owned(); + if let Some(site) = surrogate_path(&item, format!("{path}[{key_ascii}]"))? { + return Ok(Some(site)); + } + } + return Ok(None); + } + if value.is_instance_of::() || value.is_instance_of::() { + for (index, item) in value.try_iter()?.enumerate() { + if let Some(site) = surrogate_path(&item?, format!("{path}[{index}]"))? { + return Ok(Some(site)); + } + } + return Ok(None); + } + let Ok(text) = value.downcast::() else { + return Ok(None); + }; + Ok(first_surrogate_offset(text)?.map(|offset| (path, offset))) +} + +/// The UTF-8 byte offset of the first lone surrogate in `text`. +fn first_surrogate_offset(text: &Bound<'_, PyString>) -> PyResult> { + let encoded = text.call_method1(ENCODE, (UTF32_LE, SURROGATE_PASS))?; + let mut offset = 0; + for unit in encoded.extract::>()?.as_chunks::<4>().0 { + let code_point = u32::from_le_bytes([unit[0], unit[1], unit[2], unit[3]]); + match char::from_u32(code_point) { + Some(character) => offset += character.len_utf8(), + None => return Ok(Some(offset)), + } + } + Ok(None) +} + /// Return `(resolved vars, reads)` as `expand_effective_vars` would, or `(error, reads)`. #[pyfunction] fn expand_effective_vars<'py>( @@ -382,5 +448,6 @@ pub(crate) fn expanded<'py>( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(expand_config_templates, module)?)?; module.add_function(wrap_pyfunction!(expand_effective_vars, module)?)?; + module.add_function(wrap_pyfunction!(lone_surrogate_site, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/deferred_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/deferred_targets.rs new file mode 100644 index 0000000000..f75e461918 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/deferred_targets.rs @@ -0,0 +1,27 @@ +//! Deferred target schema and database templates split natively around known variables. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_model_config::templates::main::deferred_target_segments::deferred_target_segments as segments; +use sqlbuild_model_config::templates::models::DeferredSegment; + +/// `(is_variable, text)` pieces of a deferred target field after context substitution. +#[pyfunction] +fn deferred_target_segments( + template: &str, + logical: &str, + context_keys: Vec, + variables: Vec, +) -> Vec<(bool, String)> { + segments(template, logical, &context_keys, &variables) + .into_iter() + .map(|segment| match segment { + DeferredSegment::Text(text) => (false, text), + DeferredSegment::Variable(name) => (true, name), + }) + .collect() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(deferred_target_segments, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index 4bbf0ded66..56e7cb86ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -1,22 +1,22 @@ -//! MODEL header columns and audits parsed natively into the Python contract objects. +//! Parsed MODEL header columns and audits built into the Python contract objects. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; +use pyo3::{FromPyObject, IntoPyObject}; use sqlbuild_model_config::header_metadata::models::{ ParsedAudit, ParsedColumn, ParsedThresholds, ThresholdBound, }; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::model_config::config_errors::native_config_error; /// The Python classes the parsed metadata becomes, read from a Python mapping. #[derive(FromPyObject)] #[pyo3(from_item_all)] -struct ContractClasses<'py> { - schema_column: Bound<'py, PyAny>, - schema_audit_instance: Bound<'py, PyAny>, +pub(crate) struct ContractClasses<'py> { + pub(crate) schema_column: Bound<'py, PyAny>, + pub(crate) schema_audit_instance: Bound<'py, PyAny>, + pub(crate) schema_dynamic_column_family: Bound<'py, PyAny>, + pub(crate) source_location: Bound<'py, PyAny>, measurement_thresholds: Bound<'py, PyAny>, measurement_threshold_bound: Bound<'py, PyAny>, severities: Bound<'py, PyDict>, @@ -24,31 +24,7 @@ struct ContractClasses<'py> { allocate: Bound<'py, PyAny>, } -/// A model's columns and audits as contract tuples or errors; no audits after a column error. -#[pyfunction] -fn parse_model_header_metadata<'py>( - py: Python<'py>, - header: (Bound<'py, PyAny>, Bound<'py, PyAny>), - source: (Bound<'py, PyDict>, String), - classes: ContractClasses<'py>, -) -> PyResult> { - let (columns, audits) = header; - let (locations, path) = source; - let path: &str = &path; - let metadata = parse_header_metadata(&PyNode(columns), &PyNode(audits), path); - let columns = match metadata.columns { - Err(error) => { - return (native_config_error(py, error)?, py.None()).into_pyobject(py); - } - Ok(columns) => column_tuple(py, &classes, &columns, &locations)?, - }; - match metadata.audits { - Err(error) => (columns, native_config_error(py, error)?).into_pyobject(py), - Ok(audits) => (columns, audit_tuple(py, &classes, &audits)?).into_pyobject(py), - } -} - -fn column_tuple<'py>( +pub(crate) fn column_tuple<'py>( py: Python<'py>, classes: &ContractClasses<'py>, parsed: &[ParsedColumn>], @@ -61,7 +37,7 @@ fn column_tuple<'py>( PyTuple::new(py, columns) } -fn audit_tuple<'py>( +pub(crate) fn audit_tuple<'py>( py: Python<'py>, classes: &ContractClasses<'py>, parsed: &[ParsedAudit>], @@ -171,7 +147,8 @@ fn audit_instance<'py>( ) } -fn thresholds<'py>( +/// `MeasurementThresholds` of `parsed`, or `None`. +pub(crate) fn thresholds<'py>( py: Python<'py>, classes: &ContractClasses<'py>, parsed: Option<&ParsedThresholds>, @@ -217,8 +194,3 @@ fn threshold_bound<'py>( ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_model_header_metadata, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index 72e2c180e5..cf2db8c188 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -1,6 +1,8 @@ pub(crate) mod authored_nodes; +pub(crate) mod column_migrations; pub(crate) mod config_errors; pub(crate) mod config_templates; +pub(crate) mod deferred_targets; pub(crate) mod errors; pub(crate) mod header_metadata; pub(crate) mod model_config_builder; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index ba123881c9..93ea8131d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -6,7 +6,7 @@ use pyo3::types::{ PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods, }; -use pyo3::{IntoPyObject, PyErr, pyclass, pymethods}; +use pyo3::{PyErr, pyclass, pymethods}; use sqlbuild_core::text::main::python_text::python_text; use sqlbuild_core::text::models::PythonText; use sqlbuild_model_config::config_presence::main::contains_template::contains_template; @@ -22,11 +22,13 @@ use sqlbuild_model_config::templates::models::TemplateOptions; use sqlbuild_model_config::types::AuthoredNode; use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, built_config}; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::config_templates::{ PythonHost, Stop, TemplateSources, expanded, template_error, }; +use crate::bindings::_helpers::model_config::errors::NativeConfigError; const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; /// Hook keys in the sorted order `validate_model_hook_config` checks them. @@ -99,15 +101,21 @@ pub(crate) struct NativeModelConfigBuilder { } /// One model's built config before Python wraps it in `CompileModelConfig`. -struct BuiltConfig<'py> { - values: Bound<'py, PyDict>, - header_keys: Vec, - logical_schema: Option>, - layer_schema_configured: bool, - logical_database: Option>, - retention: RetentionOverride, - table_type: TableTypeOverride, - reads: Vec<(&'static str, String)>, +pub(crate) struct BuiltConfig<'py> { + pub(crate) values: Bound<'py, PyDict>, + pub(crate) header_keys: Vec, + pub(crate) logical_schema: Option>, + pub(crate) layer_schema_configured: bool, + pub(crate) logical_database: Option>, + pub(crate) retention: RetentionOverride, + pub(crate) table_type: TableTypeOverride, + pub(crate) reads: Vec<(&'static str, String)>, +} + +/// A native build's outcome: the built config, or the config error it raises. +pub(crate) enum ConfigBuild<'py> { + Built(BuiltConfig<'py>), + Rejected(Py), } /// `((target name, run id), (database, schema) or None)` of the active target. @@ -155,35 +163,139 @@ impl NativeModelConfigBuilder { }) } - /// Return the selected path-default key, `None`, or the conflict error Python raises. - fn path_default(&self, py: Python<'_>, model_path: &str) -> PyResult> { - compiler_guard(|| { - Ok(match select_path_default(model_path, &self.path_keys) { - PathDefaultChoice::Selected(key) => key.into_pyobject(py)?.into_any().unbind(), - PathDefaultChoice::Conflict(error) => native_config_error(py, error)?.into_any(), - }) - }) - } - - /// Return the built config parts or the first config error. - fn build<'py>( + /// Return the model's `CompileModelConfig` or the first config error. + fn model_config<'py>( &self, py: Python<'py>, header: Bound<'py, PyDict>, - matched_path_default: Option<&str>, - model_name: &str, + model: (Option, String), + classes: ConfigClasses<'py>, ) -> PyResult> { + let (matched_path_default, model_name) = model; + let matched_path_default: Option<&str> = matched_path_default.as_deref(); compiler_guard( - || match self.build_config(py, &header, matched_path_default, model_name) { - Ok(built) => built_tuple(py, built).map(|built| built.into_any().unbind()), - Err(Halt::Config(error)) => Ok(native_config_error(py, error)?.into_any()), - Err(Halt::Error(error)) => Err(error), + || match self.built(py, &header, matched_path_default, &model_name)? { + ConfigBuild::Built(built) => { + Ok(built_config(&classes, &built, matched_path_default)?.unbind()) + } + ConfigBuild::Rejected(error) => Ok(error.into_any()), }, ) } } impl NativeModelConfigBuilder { + /// The selected path-default key, or the conflict error Python raises. + pub(crate) fn selected_path_default( + &self, + py: Python<'_>, + model_path: &str, + ) -> PyResult, Py>> { + Ok(match select_path_default(model_path, &self.path_keys) { + PathDefaultChoice::Selected(key) => Ok(key), + PathDefaultChoice::Conflict(error) => Err(native_config_error(py, error)?), + }) + } + + /// Build one model's config natively. + pub(crate) fn built<'py>( + &self, + py: Python<'py>, + header: &Bound<'py, PyDict>, + matched_path_default: Option<&str>, + model_name: &str, + ) -> PyResult> { + match self.build_config(py, header, matched_path_default, model_name) { + Ok(built) => Ok(ConfigBuild::Built(built)), + Err(Halt::Config(error)) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), + Err(Halt::Error(error)) => Err(error), + } + } + + /// `build_layered_model_values` with an empty header: what a path default's models inherit. + pub(crate) fn inherited_values<'py>( + &self, + py: Python<'py>, + matched_path_default: Option<&str>, + ) -> PyResult, Py>> { + let layered = (|| -> Built> { + let mut layered = self.project_defaults(py)?; + if let Some(key) = matched_path_default { + layered = merged_layer(py, &layered, &self.path_default_values(py, key)?)?; + } + merged_layer(py, &layered, &PyDict::new(py)) + })(); + match layered { + Ok(values) => Ok(Ok(values)), + Err(Halt::Config(error)) => Ok(Err(native_config_error(py, error)?)), + Err(Halt::Error(error)) => Err(error), + } + } + + /// `build_run_context_values`: the run's id and target. + fn run_context_values<'py>(&self, py: Python<'py>) -> PyResult> { + let context = PyDict::new(py); + context.set_item(RUN_ID_CONTEXT, self.run_id.bind(py))?; + context.set_item( + RUN_TARGET_CONTEXT, + self.target_name + .as_ref() + .map_or_else(|| py.None().into_bound(py), |item| item.bind(py).clone()), + )?; + Ok(context) + } + + /// `build_model_context_values` for a model's config values. + pub(crate) fn model_context_values<'py>( + &self, + values: &Bound<'py, PyDict>, + model_name: &str, + include_target_values: bool, + ) -> PyResult> { + let py: Python<'py> = values.py(); + let none = py.None().into_bound(py); + let database = string_entry(values, DATABASE_KEY)?; + let schema = string_entry(values, SCHEMA_KEY)?; + let alias = match string_entry(values, ALIAS_KEY)? { + Some(alias) => alias, + None => PyString::new(py, model_name).into_any(), + }; + let context = self.run_context_values(py)?; + context.set_item(MODEL_NAME_CONTEXT, model_name)?; + context.set_item(MODEL_DATABASE_CONTEXT, database.as_ref().unwrap_or(&none))?; + context.set_item(MODEL_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; + context.set_item(MODEL_ALIAS_CONTEXT, &alias)?; + if !include_target_values { + return Ok(context); + } + let qualified = match (&database, &schema) { + (Some(database), Some(schema)) => Some(format!( + "{}.{}.{}", + text_of(database)?, + text_of(schema)?, + text_of(&alias)? + )), + (None, Some(schema)) => Some(format!("{}.{}", text_of(schema)?, text_of(&alias)?)), + _ => None, + }; + context.set_item( + DESTINATION_DATABASE_CONTEXT, + database.as_ref().unwrap_or(&none), + )?; + context.set_item(DESTINATION_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; + context.set_item(DESTINATION_TABLE_CONTEXT, &alias)?; + context.set_item(DESTINATION_QUALIFIED_CONTEXT, qualified)?; + Ok(context) + } + + /// The active target's `(database, schema)`, or `(None, None)` without a target. + pub(crate) fn target_namespace<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.target_namespace { + Some((database, schema)) => PyTuple::new(py, [database.bind(py), schema.bind(py)]), + None => PyTuple::new(py, [py.None(), py.None()]), + } + } + fn build_config<'py>( &self, py: Python<'py>, @@ -393,20 +505,7 @@ impl<'py> Expansion<'_, 'py> { } fn run_context(&self) -> Built> { - let context = PyDict::new(self.py); - context.set_item(RUN_ID_CONTEXT, self.builder.run_id.bind(self.py))?; - context.set_item( - RUN_TARGET_CONTEXT, - self.optional(self.builder.target_name.as_ref()), - )?; - Ok(context) - } - - fn optional(&self, value: Option<&Py>) -> Bound<'py, PyAny> { - value.map_or_else( - || self.py.None().into_bound(self.py), - |item| item.bind(self.py).clone(), - ) + Ok(self.builder.run_context_values(self.py)?) } /// Return `build_model_context_values` for these values. @@ -415,40 +514,9 @@ impl<'py> Expansion<'_, 'py> { values: &Bound<'py, PyDict>, include_target_values: bool, ) -> Built> { - let py = self.py; - let none = py.None().into_bound(py); - let database = string_entry(values, DATABASE_KEY)?; - let schema = string_entry(values, SCHEMA_KEY)?; - let alias = match string_entry(values, ALIAS_KEY)? { - Some(alias) => alias, - None => PyString::new(py, self.model_name).into_any(), - }; - let context = self.run_context()?; - context.set_item(MODEL_NAME_CONTEXT, self.model_name)?; - context.set_item(MODEL_DATABASE_CONTEXT, database.as_ref().unwrap_or(&none))?; - context.set_item(MODEL_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; - context.set_item(MODEL_ALIAS_CONTEXT, &alias)?; - if !include_target_values { - return Ok(context); - } - let qualified = match (&database, &schema) { - (Some(database), Some(schema)) => Some(format!( - "{}.{}.{}", - text_of(database)?, - text_of(schema)?, - text_of(&alias)? - )), - (None, Some(schema)) => Some(format!("{}.{}", text_of(schema)?, text_of(&alias)?)), - _ => None, - }; - context.set_item( - DESTINATION_DATABASE_CONTEXT, - database.as_ref().unwrap_or(&none), - )?; - context.set_item(DESTINATION_SCHEMA_CONTEXT, schema.as_ref().unwrap_or(&none))?; - context.set_item(DESTINATION_TABLE_CONTEXT, &alias)?; - context.set_item(DESTINATION_QUALIFIED_CONTEXT, qualified)?; - Ok(context) + Ok(self + .builder + .model_context_values(values, self.model_name, include_target_values)?) } /// Check preserved namespaces and apply `apply_environment_database_schema_overrides`. @@ -615,18 +683,17 @@ fn set_entry<'py>(values: &Bound<'py, PyDict>, key: &str) -> Built(values: &Bound<'py, PyDict>, key: &str) -> Built>> { +fn string_entry<'py>( + values: &Bound<'py, PyDict>, + key: &str, +) -> PyResult>> { Ok(values .get_item(key)? .filter(|value| value.is_instance_of::())) } -fn text_of(value: &Bound<'_, PyAny>) -> Built { - Ok(value - .downcast::() - .map_err(PyErr::from)? - .to_string_lossy() - .into_owned()) +fn text_of(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.downcast::()?.to_string_lossy().into_owned()) } fn sequence_items<'py>(value: &Bound<'py, PyAny>) -> Option>> { @@ -711,31 +778,6 @@ fn sorted_keys(header: &Bound<'_, PyDict>) -> Built> { Ok(keys) } -fn built_tuple<'py>(py: Python<'py>, built: BuiltConfig<'py>) -> PyResult> { - let retention = match built.retention { - RetentionOverride::Inherit => None, - RetentionOverride::Unmanaged => Some((None, true)), - RetentionOverride::Days(days) => Some((Some(days), false)), - }; - let table_type = match built.table_type { - TableTypeOverride::Inherit => None, - TableTypeOverride::Permanent => Some("permanent"), - TableTypeOverride::Transient => Some("transient"), - }; - ( - built.values, - PyTuple::new(py, built.header_keys)?, - ( - built.logical_schema, - built.layer_schema_configured, - built.logical_database, - ), - (retention, table_type), - built.reads, - ) - .into_pyobject(py) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs index a31bea5e84..bae097d9ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -6,6 +6,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{IntoPyObject, pyclass, pymethods}; +use sqlbuild_attachments::functions::models::KnownResources; use sqlbuild_core::text::main::python_text::python_text; use sqlbuild_model_config::model_validation::main::validate_model_config::validate_model_config; use sqlbuild_model_config::model_validation::main::validate_model_references::validate_model_references; @@ -35,6 +36,19 @@ pub(crate) struct NativeModelValidator { project: ProjectValidationFacts, } +impl NativeModelValidator { + /// The project's discovered resource names, which every reference resolves against. + pub(crate) fn known_resources(&self) -> KnownResources<'_> { + KnownResources { + models: &self.project.models, + seeds: &self.project.seeds, + sources: &self.project.sources, + functions: &self.project.functions, + table_functions: &self.project.table_functions, + } + } +} + #[pymethods] impl NativeModelValidator { #[new] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index 4414cbafb1..b6ea38d056 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -3,13 +3,15 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ - config_errors, config_templates, header_metadata, model_config_builder, model_validator, + column_migrations, config_errors, config_templates, deferred_targets, model_config_builder, + model_validator, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + column_migrations::register(module)?; config_errors::register(module)?; - header_metadata::register(module)?; config_templates::register(module)?; + deferred_targets::register(module)?; model_config_builder::register(module)?; model_validator::register(module)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs index 9e1646f00c..0bce593838 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/resources.rs @@ -10,6 +10,7 @@ use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use sqlbuild_analysis::assembly::project::errors::AssemblyError; use sqlbuild_analysis::assembly::project::main::assemble_project_resources::assemble_project_resources; use sqlbuild_analysis::assembly::project::main::check_sql_syntax::check_sql_syntax; +use sqlbuild_analysis::assembly::project::main::environment_template_names::environment_template_names as template_names; use sqlbuild_analysis::assembly::project::main::sql_syntax_error::sql_syntax_error as check_syntax_error; use sqlbuild_analysis::assembly::project::models::{ AuditFacts, FunctionFacts, InputRead, ModelFacts, Namespace, OpaqueRender, OpaqueVariable, @@ -145,7 +146,17 @@ fn sql_syntax_error( } else { SyntaxMode::Validate }; - py.compiler_detach(|| Ok(check_syntax_error(&check, &dialect, mode))) + syntax_error_message(py, &check, &dialect, mode) +} + +/// Python's syntax error message for one check, raising what Python's validation raises. +pub(crate) fn syntax_error_message( + py: Python<'_>, + check: &SyntaxCheck, + dialect: &str, + mode: SyntaxMode, +) -> PyResult> { + py.compiler_detach(|| Ok(check_syntax_error(check, dialect, mode))) .map_err(compiler_error)? .map_err(syntax_value_error) } @@ -171,7 +182,16 @@ fn check_native_sql_syntax( .into_iter() .map(|(sql, placeholders)| SyntaxCheck { sql, placeholders }) .collect(); - py.compiler_detach(|| check_sql_syntax(&dialect, &checks)) + syntax_valid(py, &dialect, &checks) +} + +/// Whether every check parses, raising what Python's validation raises. +pub(crate) fn syntax_valid( + py: Python<'_>, + dialect: &str, + checks: &[SyntaxCheck], +) -> PyResult { + py.compiler_detach(|| check_sql_syntax(dialect, checks)) .map_err(|reason| compiler_error(native_failure(SYNTAX_CHECK_CONTEXT, &reason)))? .map_err(syntax_value_error) } @@ -371,9 +391,16 @@ fn namespace_row(namespace: Namespace) -> NamespaceRow { ) } +/// The distinct `ENV:` names target templates read, in first-seen order. +#[pyfunction] +fn environment_template_names(values: Vec>) -> Vec { + template_names(&values) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(assemble_project_resource_facts, module)?)?; module.add_function(wrap_pyfunction!(check_native_sql_syntax, module)?)?; module.add_function(wrap_pyfunction!(sql_syntax_error, module)?)?; + module.add_function(wrap_pyfunction!(environment_template_names, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs new file mode 100644 index 0000000000..400ec68178 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_edits.rs @@ -0,0 +1,91 @@ +//! Contract adoption's MODEL header and YAML source edits, computed natively. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_refactor::refactoring::errors::{RefactorError, RefactorErrorKind}; +use sqlbuild_refactor::refactoring::main::edit_model_contract::edit_model_contract; +use sqlbuild_refactor::refactoring::main::edit_source_contract::edit_source_contract; +use sqlbuild_refactor::refactoring::models::{ + ContractEdit, ContractModelColumn, ContractModelRequest, ContractSourceRequest, +}; +use sqlbuild_refactor::refactoring::types::PhysicalColumn; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +/// `(edited contents, ownership conflict, ContractAdoptionError message)`. +type EditRow = (Option, Option, Option); +type ModelColumnRow = (String, Option, Option); +type ModelRow = ( + String, + String, + bool, + Vec, + Vec>, + Vec, + bool, +); +type SourceRow = ( + String, + String, + String, + Vec<(String, Option)>, + Vec, + bool, +); + +fn edit_row(result: Result) -> PyResult { + match result { + Ok(ContractEdit::Edited(contents)) => Ok((Some(contents), None, None)), + Ok(ContractEdit::Conflict(conflict)) => Ok((None, Some(conflict), None)), + Err(error) if error.kind == RefactorErrorKind::Edit => { + Ok((None, None, Some(error.message))) + } + Err(error) => Err(PyValueError::new_err(error.message)), + } +} + +/// Python's `edit_model` over the model facts a compiled view holds. +#[pyfunction] +fn contract_model_edit(row: ModelRow) -> PyResult { + let (contents, relative_path, shared_schema, columns, dynamic_patterns, physical, overwrite) = + row; + let request = ContractModelRequest { + contents, + relative_path, + shared_schema, + columns: columns + .into_iter() + .map(|(name, declared_type, location_path)| ContractModelColumn { + name, + declared_type, + location_path, + }) + .collect(), + dynamic_patterns, + physical_columns: physical, + overwrite, + }; + compiler_guard(|| edit_row(edit_model_contract(&request))) +} + +/// Python's `edit_source` over the source facts a compiled view holds. +#[pyfunction] +fn contract_source_edit(row: SourceRow) -> PyResult { + let (contents, discovered_contents, name, columns, physical, overwrite) = row; + let request = ContractSourceRequest { + contents, + discovered_contents, + name, + columns, + physical_columns: physical, + overwrite, + }; + compiler_guard(|| edit_row(Ok(edit_source_contract(&request)))) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(contract_model_edit, module)?)?; + module.add_function(wrap_pyfunction!(contract_source_edit, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs new file mode 100644 index 0000000000..19f416b5bb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod contract_edits; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs new file mode 100644 index 0000000000..fa4ab9dc4a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/registration.rs @@ -0,0 +1,148 @@ +//! JSON bindings for native `sqb rename` and `sqb mv`; expected failures return `{"error": ...}`. + +use std::cell::RefCell; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::json; +use sqlbuild_refactor::refactoring::errors::{RefactorError, RefactorErrorKind}; +use sqlbuild_refactor::refactoring::main::commit_refactor_plan::commit_refactor_plan; +use sqlbuild_refactor::refactoring::main::plan_refactor::plan_refactor; +use sqlbuild_refactor::refactoring::main::stage_refactor_plan::stage_refactor_plan; +use sqlbuild_refactor::refactoring::main::with_model_migration::with_model_migration; +use sqlbuild_refactor::refactoring::models::{ + DeclarationPlacement, ModelFacts, RefactorFacts, RefactorPlan, RefactorRequest, +}; +use sqlbuild_refactor::refactoring::types::Originals; + +use crate::bindings::_helpers::boundary::panics::{ + NativeCompilerError, compiler_guard, value_error, +}; + +fn decode(text: &str) -> PyResult { + serde_json::from_str(text).map_err(value_error) +} + +fn encode(key: &str, result: Result) -> PyResult { + let value = match result { + Ok(value) => json!({ key: value }), + Err(error) if error.kind == RefactorErrorKind::Internal => { + return Err(NativeCompilerError::new_err(error.message)); + } + Err(error) => json!({ "error": error }), + }; + serde_json::to_string(&value).map_err(value_error) +} + +/// Plan one refactoring; `declaration_placement((model, destination))` returns JSON placement. +#[pyfunction] +fn plan_refactor_json( + py: Python<'_>, + facts_json: &str, + request_json: &str, + declaration_placement: Py, +) -> PyResult { + compiler_guard(|| { + let facts: RefactorFacts = decode(facts_json)?; + let request: RefactorRequest = decode(request_json)?; + let host_error: RefCell> = RefCell::new(None); + let host = |model: &str, destination: &str| { + let placement = declaration_placement + .call1(py, ((model, destination),)) + .and_then(|value| value.extract::(py)) + .and_then(|text| decode::(&text)); + placement.map_err(|error| { + host_error.replace(Some(error)); + RefactorError::internal("the declaration-move host failed") + }) + }; + let result = plan_refactor(&facts, &request, &host); + if let Some(error) = host_error.into_inner() { + return Err(error); + } + encode("plan", result) + }) +} + +/// Copy the project inputs once, apply the plan in `staging_dir`, and return the originals. +#[pyfunction] +fn stage_refactor_plan_json( + project_dir: &str, + staging_dir: &str, + plan_json: &str, + copy_inputs: bool, +) -> PyResult { + compiler_guard(|| { + let plan: RefactorPlan = decode(plan_json)?; + encode( + "originals", + stage_refactor_plan( + Path::new(project_dir), + Path::new(staging_dir), + &plan, + copy_inputs, + ), + ) + }) +} + +/// The plan with model `migrate_from` added whenever the relation moves. +#[pyfunction] +#[pyo3(signature = (plan_json, before_json, after_json, originals_json))] +fn with_model_migration_json( + plan_json: &str, + before_json: &str, + after_json: Option<&str>, + originals_json: &str, +) -> PyResult { + compiler_guard(|| { + let plan: RefactorPlan = decode(plan_json)?; + let before: Vec = decode(before_json)?; + let after: Option> = after_json.map(decode).transpose()?; + let originals: Originals = decode(originals_json)?; + encode( + "plan", + Ok(with_model_migration( + &plan, + &before, + after.as_deref(), + &originals, + )), + ) + }) +} + +/// Write a verified plan into the project, restoring every file if any write fails. +#[pyfunction] +fn commit_refactor_plan_json( + project_dir: &str, + originals_json: &str, + plan_json: &str, +) -> PyResult { + compiler_guard(|| { + let plan: RefactorPlan = decode(plan_json)?; + let originals: Originals = decode(originals_json)?; + let written = + commit_refactor_plan(Path::new(project_dir), &originals, &plan).map(path_strings); + encode("written", written) + }) +} + +fn path_strings(paths: Vec) -> Vec { + paths + .into_iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_refactor_json, module)?)?; + module.add_function(wrap_pyfunction!(stage_refactor_plan_json, module)?)?; + module.add_function(wrap_pyfunction!(with_model_migration_json, module)?)?; + module.add_function(wrap_pyfunction!(commit_refactor_plan_json, module)?)?; + crate::bindings::_helpers::refactor::contract_edits::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs new file mode 100644 index 0000000000..8cfb32f4c2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -0,0 +1,326 @@ +//! Enum and constant references expanded natively, for model queries and every other caller. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyInt, PyTuple}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiled_project::errors::DeclarationError; +use sqlbuild_analysis::compiled_project::main::declaration_error_message::declaration_error_message; +use sqlbuild_analysis::compiled_project::main::enum_member_literal::enum_member_literal; +use sqlbuild_analysis::compiled_project::main::splice_declarations::splice_declarations; +use sqlbuild_analysis::compiled_project::models::{ + DeclarationSplice, InaccessibleDeclaration, SplicedSql, +}; + +const ENUM_REFERENCE_KIND: u8 = 0; +const ENUM_KIND: &str = "enum"; +const CONSTANT_KIND: &str = "constant"; +const ENUMS: &str = "enums"; +const CONSTANTS: &str = "constants"; +const INACCESSIBLE_ENUMS: &str = "inaccessible_enums"; +const INACCESSIBLE_CONSTANTS: &str = "inaccessible_constants"; +const ENUM_VISIBILITY: &str = "enum_visibility"; +const CONSTANT_VISIBILITY: &str = "constant_visibility"; +const MEMBERS: &str = "members"; +const NAME: &str = "name"; +const VALUE: &str = "value"; +const KIND: &str = "kind"; +const PATH: &str = "path"; +const LINE: &str = "line"; +const COLUMN: &str = "column"; +const SCOPE: &str = "scope"; +const OWNING_PATH: &str = "owning_path"; +const OWNERSHIP_ROOT: &str = "ownership_root"; +const IDENTITY: &str = "identity"; +const OWNER: &str = "owner"; +const DECLARATION: &str = "declaration"; +const THROUGH: &str = "through"; +const CONSUMER: &str = "consumer"; +const ENUM_MEMBER: &str = "enum_member"; + +/// `(kind, name, member, start, end)` for one natively scanned reference. +pub(crate) type ScannedReference = (u8, String, Option, usize, usize); +/// `(source start, source end, output start, output end)`. +pub(crate) type SpanRow = (usize, usize, usize, usize); + +/// The Python classes expansion builds usages and errors from. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct ExpansionClasses<'py> { + pub(crate) usage_record: Bound<'py, PyAny>, + pub(crate) compile_error: Bound<'py, PyAny>, +} + +/// Who reads the declarations, whether it prefers relationship visibility, and its error path. +pub(crate) struct ExpansionConsumer<'a, 'py> { + pub(crate) identity: Option>, + pub(crate) relationship_visibility: bool, + pub(crate) path: &'a str, +} + +/// One expansion: the SQL, its scan, the visible declarations and the adapter's constant renderer. +pub(crate) struct DeclarationExpansion<'a, 'py> { + pub(crate) sql: &'a str, + pub(crate) references: &'a [ScannedReference], + pub(crate) stop: Option, + pub(crate) context: &'a Bound<'py, PyAny>, + pub(crate) consumer: ExpansionConsumer<'a, 'py>, + pub(crate) render_constant: &'a Bound<'py, PyAny>, + pub(crate) classes: &'a ExpansionClasses<'py>, +} + +/// The expanded SQL with its spans, and the usage records it adds in reference order. +pub(crate) struct ExpandedDeclarations<'py> { + pub(crate) spliced: SplicedSql, + pub(crate) usages: Vec>, +} + +/// Splice each reference in order, raising the attachment's error for the first bad one. +pub(crate) fn expand_declarations<'py>( + request: &DeclarationExpansion<'_, 'py>, +) -> PyResult> { + let mut splices: Vec = Vec::with_capacity(request.references.len()); + let mut usages: Vec> = Vec::new(); + for (kind, name, member, start, end) in request.references { + let (replacement, visibility) = if *kind == ENUM_REFERENCE_KIND { + enum_text(request, name, member.as_deref())? + } else { + constant_text(request, name)? + }; + if let Some(consumer) = &request.consumer.identity { + for record in usage_visibility(&visibility, request.consumer.relationship_visibility)? { + usages.push(usage( + request.classes, + consumer, + &record, + member.as_deref(), + )?); + } + } + splices.push(DeclarationSplice { + start: *start, + end: *end, + replacement, + }); + } + if let Some(code) = request.stop { + return Err(compile_error(request, &DeclarationError::ScanStop(code))); + } + Ok(ExpandedDeclarations { + spliced: splice_declarations(request.sql, &splices), + usages, + }) +} + +fn enum_text<'py>( + request: &DeclarationExpansion<'_, 'py>, + name: &str, + member: Option<&str>, +) -> PyResult<(String, Bound<'py, PyAny>)> { + let declaration: Bound<'py, PyAny> = visible_declaration(request, ENUM_KIND, name)?; + let member: &str = member.unwrap_or_default(); + let mut members: Vec = Vec::new(); + for candidate in declaration.getattr(MEMBERS)?.try_iter()? { + let candidate: Bound<'py, PyAny> = candidate?; + let candidate_name: String = candidate.getattr(NAME)?.extract()?; + if candidate_name == member { + let value: Bound<'py, PyAny> = candidate.getattr(VALUE)?; + let literal: String = + enum_member_literal(&value.str()?.to_string(), value.is_instance_of::()); + return Ok((literal, visibility(request, ENUM_VISIBILITY, name)?)); + } + members.push(candidate_name); + } + Err(compile_error( + request, + &DeclarationError::UnknownMember { + name: name.to_owned(), + member: member.to_owned(), + members, + }, + )) +} + +fn constant_text<'py>( + request: &DeclarationExpansion<'_, 'py>, + name: &str, +) -> PyResult<(String, Bound<'py, PyAny>)> { + let declaration: Bound<'py, PyAny> = visible_declaration(request, CONSTANT_KIND, name)?; + let arguments: Bound<'py, PyDict> = PyDict::new(declaration.py()); + arguments.set_item(DECLARATION, declaration)?; + let rendered: String = request + .render_constant + .call((), Some(&arguments))? + .extract()?; + Ok((rendered, visibility(request, CONSTANT_VISIBILITY, name)?)) +} + +/// The visible declaration, or the unknown or inaccessible error for it. +fn visible_declaration<'py>( + request: &DeclarationExpansion<'_, 'py>, + kind: &str, + name: &str, +) -> PyResult> { + let (visible_attribute, inaccessible_attribute) = if kind == ENUM_KIND { + (ENUMS, INACCESSIBLE_ENUMS) + } else { + (CONSTANTS, INACCESSIBLE_CONSTANTS) + }; + let visible: Bound<'py, PyDict> = request + .context + .getattr(visible_attribute)? + .downcast_into()?; + if let Some(declaration) = visible.get_item(name)? { + return Ok(declaration); + } + let inaccessible: Bound<'py, PyDict> = request + .context + .getattr(inaccessible_attribute)? + .downcast_into()?; + let error: DeclarationError = match inaccessible.get_item(name)? { + Some(record) => DeclarationError::Inaccessible { + kind: kind.to_owned(), + name: name.to_owned(), + record: inaccessible_record(&record)?, + }, + None => DeclarationError::Unknown { + kind: kind.to_owned(), + name: name.to_owned(), + visible: visible.keys().extract()?, + }, + }; + Err(compile_error(request, &error)) +} + +fn inaccessible_record(record: &Bound<'_, PyAny>) -> PyResult { + let owning_path: Bound<'_, PyAny> = record.getattr(OWNING_PATH)?; + let owner: String = if owning_path.is_truthy()? { + owning_path.str()?.to_string() + } else { + record + .getattr(OWNERSHIP_ROOT)? + .getattr(PATH)? + .str()? + .to_string() + }; + let owner_resource: Bound<'_, PyAny> = record.getattr(IDENTITY)?.getattr(OWNER)?; + let owner_identity: Option<(String, String)> = if owner_resource.is_none() { + None + } else { + Some(( + owner_resource + .getattr(KIND)? + .getattr(VALUE)? + .str()? + .to_string(), + owner_resource.getattr(NAME)?.str()?.to_string(), + )) + }; + Ok(InaccessibleDeclaration { + path: record.getattr(PATH)?.str()?.to_string(), + line: record.getattr(LINE)?.extract()?, + column: record.getattr(COLUMN)?.extract()?, + scope: record.getattr(SCOPE)?.getattr(VALUE)?.str()?.to_string(), + owner, + owner_identity, + }) +} + +fn visibility<'py>( + request: &DeclarationExpansion<'_, 'py>, + attribute: &str, + name: &str, +) -> PyResult> { + let records: Bound<'py, PyDict> = request.context.getattr(attribute)?.downcast_into()?; + Ok(records + .get_item(name)? + .unwrap_or_else(|| PyTuple::empty(records.py()).into_any())) +} + +/// Tests and scenarios record relationship visibility when they have any; others record all. +pub(crate) fn usage_visibility<'py>( + visibility: &Bound<'py, PyAny>, + relationship_visibility: bool, +) -> PyResult>> { + let records: Vec> = visibility + .try_iter()? + .collect::>>>()?; + if !relationship_visibility { + return Ok(records); + } + let mut related: Vec> = Vec::new(); + for record in &records { + if !record.getattr(THROUGH)?.is_none() { + related.push(record.clone()); + } + } + Ok(if related.is_empty() { records } else { related }) +} + +fn usage<'py>( + classes: &ExpansionClasses<'py>, + consumer: &Bound<'py, PyAny>, + record: &Bound<'py, PyAny>, + member: Option<&str>, +) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(consumer.py()); + fields.set_item(CONSUMER, consumer)?; + fields.set_item(DECLARATION, record.getattr(DECLARATION)?)?; + fields.set_item(THROUGH, record.getattr(THROUGH)?)?; + fields.set_item(ENUM_MEMBER, member)?; + classes.usage_record.call((), Some(&fields)) +} + +fn compile_error(request: &DeclarationExpansion<'_, '_>, error: &DeclarationError) -> PyErr { + let message: String = declaration_error_message(error, request.consumer.path); + match request.classes.compile_error.call1((message,)) { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +pub(crate) fn span_rows(spliced: &SplicedSql) -> Vec { + spliced + .spans + .iter() + .map(|span| { + ( + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ) + }) + .collect() +} + +/// Expand one string's scanned references; `source` is `(sql, references, stop code)`. +#[pyfunction] +fn expand_declaration_references<'py>( + source: (String, Vec, Option), + context: Bound<'py, PyAny>, + consumer: (Option>, bool, String), + renderers: (Bound<'py, PyAny>, ExpansionClasses<'py>), +) -> PyResult<(String, Vec, Vec>)> { + let (sql, references, stop) = source; + let (identity, relationship_visibility, path) = consumer; + let (render_constant, classes) = renderers; + let expanded: ExpandedDeclarations<'py> = expand_declarations(&DeclarationExpansion { + sql: &sql, + references: &references, + stop, + context: &context, + consumer: ExpansionConsumer { + identity, + relationship_visibility, + path: &path, + }, + render_constant: &render_constant, + classes: &classes, + })?; + let spans: Vec = span_rows(&expanded.spliced); + Ok((expanded.spliced.sql, spans, expanded.usages)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(expand_declaration_references, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs index 9c5392a429..b416e2e2aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyList, PyListMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_render::macro_arguments::errors::ArgumentError; use sqlbuild_render::macro_arguments::main::parse_macro_arguments::parse_macro_arguments as parse; use sqlbuild_render::macro_arguments::models::{ArgumentValue, MacroArguments}; use sqlbuild_render::macro_arguments::types::ArgumentHost; @@ -66,15 +67,7 @@ fn parse_macro_arguments( text: &str, nested: Vec<(usize, usize)>, ) -> PyResult> { - let host = UnicodeHost { - unicodedata: py.import("unicodedata")?, - raised: RefCell::new(None), - }; - let parsed = parse(&host, text, &nested); - if let Some(error) = host.raised.into_inner() { - return Err(error); - } - match parsed { + match parsed_arguments(py, text, &nested)? { Ok(arguments) => arguments_row(py, &arguments), Err(error) => Ok( ("error", error.detail, error.help, error.line, error.column) @@ -85,6 +78,23 @@ fn parse_macro_arguments( } } +/// Parse a call's argument text, raising an unexpected `unicodedata` error once parsing returns. +pub(crate) fn parsed_arguments( + py: Python<'_>, + text: &str, + nested: &[(usize, usize)], +) -> PyResult> { + let host = UnicodeHost { + unicodedata: py.import("unicodedata")?, + raised: RefCell::new(None), + }; + let parsed = parse(&host, text, nested); + if let Some(error) = host.raised.into_inner() { + return Err(error); + } + Ok(parsed) +} + fn arguments_row(py: Python<'_>, arguments: &MacroArguments) -> PyResult> { let positional = values_list(py, &arguments.positional)?; let keywords = PyList::empty(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs new file mode 100644 index 0000000000..ed4f0f630d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs @@ -0,0 +1,281 @@ +//! `MacroBridge.call_class` and the class ids its memo keys recorded calls by. + +use std::collections::HashMap; + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +use sqlbuild_render::macro_calls::main::call_class_store_text::call_class_store_text; + +use crate::bindings::_helpers::render::macro_expansion_facts::ResourceRef; + +/// `(kind, name, owner (kind, name))` of a `DeclarationIdentity`, compared by value. +type IdentityKey = (String, String, Option<(String, String)>); +/// `(name, id(loaded macro), identity)`. +type NameKey = (String, usize, Option); +/// `(constants, enums)` by name and value id. +type DeclarationValues = (Vec<(String, usize)>, Vec<(String, usize)>); +/// `(id(macro context), declaration values)`. +type ContextKey = (usize, Option); +/// A name's class token and its store token, or `None` when the name is not a visible macro. +type NameToken = Option<(NameKey, Option)>; +/// A call class with the prior relations it was looked up with. +type ClassKey = ((Vec, Option), Option>); + +/// What a call's result may depend on besides its text. +pub(crate) struct CallClass { + key: (Vec, Option), + macro_store_tokens: Vec, + context_store_token: Option, + persistent: bool, +} + +/// `macro_store_token` and `context_store_token`, the value text of a stored call's inputs. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct StoreTokenClasses<'py> { + macro_token: Bound<'py, PyAny>, + context_token: Bound<'py, PyAny>, +} + +/// The call classes, class ids and tokens one compile's memo has seen. +#[derive(Debug, Default)] +pub(crate) struct CallClassState { + class_ids: HashMap, + name_tokens: HashMap<(usize, String), NameToken>, + context_tokens: HashMap<(usize, usize), (ContextKey, Option)>, + /// Owners and contexts kept alive, so the ids in keys are never reused. + retained: HashMap>, + /// Whether a persistent store is attached, so store tokens are computed. + pub(crate) store_tokens: bool, +} + +impl CallClassState { + /// Forget tokens computed before a store was attached; keep class ids. + pub(crate) fn reset_tokens(&mut self, store_tokens: bool) { + self.store_tokens = store_tokens; + self.name_tokens.clear(); + self.context_tokens.clear(); + } +} + +/// The macros one expansion resolves names from. +pub(crate) struct ClassInputs<'a, 'py> { + /// The `DeclarationResolutionContext`, or `None`. + pub(crate) declarations: &'a Bound<'py, PyAny>, + pub(crate) loaded_macros: &'a Bound<'py, PyDict>, + pub(crate) macro_context: &'a Bound<'py, PyAny>, + pub(crate) store: &'a StoreTokenClasses<'py>, +} + +impl CallClassState { + /// `call_class`: `None` when a name in the call is not a visible macro. + pub(crate) fn call_class( + &mut self, + tree_names: &[String], + inputs: &ClassInputs<'_, '_>, + ) -> PyResult> { + let scoped: bool = !inputs.declarations.is_none(); + let owner: &Bound<'_, PyAny> = if scoped { + inputs.declarations + } else { + inputs.loaded_macros.as_any() + }; + let macros: Bound<'_, PyAny> = if scoped { + inputs.declarations.getattr("macros")? + } else { + inputs.loaded_macros.clone().into_any() + }; + let _ = self + .retained + .entry(owner.as_ptr() as usize) + .or_insert_with(|| owner.clone().unbind()); + let _ = self + .retained + .entry(inputs.macro_context.as_ptr() as usize) + .or_insert_with(|| inputs.macro_context.clone().unbind()); + let mut tokens: Vec = Vec::with_capacity(tree_names.len()); + let mut store_tokens: Vec> = Vec::with_capacity(tree_names.len()); + for name in tree_names { + let Some((token, store_token)) = self.name_token((owner, ¯os), inputs, name)? + else { + return Ok(None); + }; + tokens.push(token); + store_tokens.push(store_token); + } + let mut injects_context: bool = false; + for name in tree_names { + if macros + .get_item(name)? + .getattr("injects_context")? + .is_truthy()? + { + injects_context = true; + break; + } + } + let context: Option<(ContextKey, Option)> = if injects_context { + Some(self.context_token(owner, inputs)?) + } else { + None + }; + let persistent_tokens: Vec = store_tokens.iter().flatten().cloned().collect(); + let persistent: bool = persistent_tokens.len() == store_tokens.len() + && context.as_ref().is_none_or(|(_, token)| token.is_some()); + let (context_key, context_store_token) = match context { + Some((key, token)) => (Some(key), token), + None => (None, None), + }; + Ok(Some(CallClass { + key: (tokens, context_key), + macro_store_tokens: persistent_tokens, + context_store_token, + persistent, + })) + } + + /// The class id of `call_class` with these prior relations; a new persistent class is stored. + pub(crate) fn class_id( + &mut self, + call_class: &CallClass, + prior_relations: Option<&[ResourceRef]>, + ) -> (u64, Option) { + let key: ClassKey = (call_class.key.clone(), prior_relations.map(<[_]>::to_vec)); + if let Some(found) = self.class_ids.get(&key) { + return (*found, None); + } + let id: u64 = self.class_ids.len() as u64; + let _ = self.class_ids.insert(key, id); + let class_text: Option = (self.store_tokens && call_class.persistent).then(|| { + call_class_store_text( + &call_class.macro_store_tokens, + call_class.context_store_token.as_deref(), + prior_relations, + ) + }); + (id, class_text) + } + + fn name_token( + &mut self, + scope: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + inputs: &ClassInputs<'_, '_>, + name: &str, + ) -> PyResult { + let (owner, macros) = scope; + let key: (usize, String) = (owner.as_ptr() as usize, name.to_owned()); + if let Some(found) = self.name_tokens.get(&key) { + return Ok(found.clone()); + } + let py: Python<'_> = owner.py(); + let loaded: Bound<'_, PyAny> = macros.call_method1("get", (name,))?; + let declarations: &Bound<'_, PyAny> = inputs.declarations; + let visible: bool = !loaded.is_none() + && (declarations.is_none() + || !declarations + .getattr("inaccessible_macros")? + .contains(name)?); + let token: NameToken = if visible { + let identity: Bound<'_, PyAny> = if declarations.is_none() { + py.None().into_bound(py) + } else { + declarations + .getattr("macro_records")? + .get_item(name)? + .getattr("identity")? + }; + let store_token: Option = if self.store_tokens { + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("loaded", &loaded)?; + arguments.set_item("identity", &identity)?; + inputs + .store + .macro_token + .call((), Some(&arguments))? + .extract()? + } else { + None + }; + Some(( + ( + name.to_owned(), + loaded.as_ptr() as usize, + identity_key(&identity)?, + ), + store_token, + )) + } else { + None + }; + let _ = self.name_tokens.insert(key, token.clone()); + Ok(token) + } + + fn context_token( + &mut self, + owner: &Bound<'_, PyAny>, + inputs: &ClassInputs<'_, '_>, + ) -> PyResult<(ContextKey, Option)> { + let key: (usize, usize) = ( + owner.as_ptr() as usize, + inputs.macro_context.as_ptr() as usize, + ); + if let Some(found) = self.context_tokens.get(&key) { + return Ok(found.clone()); + } + let declarations: &Bound<'_, PyAny> = inputs.declarations; + let values: Option = if declarations.is_none() { + None + } else { + Some(( + value_ids(&declarations.getattr("constants")?.downcast_into()?)?, + value_ids(&declarations.getattr("enums")?.downcast_into()?)?, + )) + }; + let store_token: Option = if self.store_tokens { + let arguments: Bound<'_, PyDict> = PyDict::new(owner.py()); + arguments.set_item("macro_context", inputs.macro_context)?; + arguments.set_item("declarations", declarations)?; + inputs + .store + .context_token + .call((), Some(&arguments))? + .extract()? + } else { + None + }; + let token: (ContextKey, Option) = ((key.1, values), store_token); + let _ = self.context_tokens.insert(key, token.clone()); + Ok(token) + } +} + +/// Each `(name, id(value))` of a declaration mapping, in insertion order. +fn value_ids(mapping: &Bound<'_, PyDict>) -> PyResult> { + mapping + .iter() + .map(|(name, value)| Ok((name.extract()?, value.as_ptr() as usize))) + .collect() +} + +fn identity_key(identity: &Bound<'_, PyAny>) -> PyResult> { + if identity.is_none() { + return Ok(None); + } + let owner: Bound<'_, PyAny> = identity.getattr("owner")?; + let owner_key: Option<(String, String)> = if owner.is_none() { + None + } else { + Some(( + owner.getattr("kind")?.getattr("value")?.extract()?, + owner.getattr("name")?.extract()?, + )) + }; + Ok(Some(( + identity.getattr("kind")?.getattr("value")?.extract()?, + identity.getattr("name")?.extract()?, + owner_key, + ))) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs index 63bec19bab..e07eeb085f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs @@ -4,13 +4,12 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::Python; use pyo3::{PyResult, pyclass, pymethods}; use sqlbuild_cache::store::models::NativeStore; -use sqlbuild_render::macro_calls::models::MacroCallEntry; use std::path::PathBuf; use std::sync::Mutex; use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; -use crate::bindings::_helpers::render::macro_call_rows::{EntryRow, event_from_row, event_row}; +use crate::bindings::_helpers::render::macro_call_classes::CallClassState; /// The native store kind holding recorded macro calls. const MACRO_CALL_STORE_KIND: &str = "macro-calls"; @@ -20,9 +19,12 @@ const MACRO_CALL_STORE_KIND: &str = "macro-calls"; #[derive(Debug, Default)] pub(crate) struct MacroCallMemo { pub(crate) inner: Mutex, + pub(crate) classes: Mutex, + /// `len(sys.modules)` when the store was last trusted, or `None` once it is not. + pub(crate) observed_modules: Mutex>, } -fn poisoned(_: T) -> pyo3::PyErr { +pub(crate) fn poisoned(_: T) -> pyo3::PyErr { PyValueError::new_err("macro call memo is poisoned") } @@ -33,37 +35,18 @@ impl MacroCallMemo { Self::default() } - /// The recorded result of a call in `class_id`, or `None` when it must execute. - fn lookup(&self, class_id: u64, call_text: &str) -> PyResult> { - let found = self - .inner + /// Forget call tokens after attaching a store; `store_tokens` when its keys are computed. + fn reset_call_classes(&self, store_tokens: bool) -> PyResult<()> { + self.classes .lock() .map_err(poisoned)? - .lookup(class_id, call_text); - Ok(found.map(|entry| { - ( - entry.sql.clone(), - entry.relations.clone(), - entry.events.iter().map(event_row).collect(), - ) - })) + .reset_tokens(store_tokens); + Ok(()) } - /// Record the `(sql, relations, events)` result of one executed call. - fn record(&self, class_id: u64, call_text: String, entry: EntryRow) -> PyResult<()> { - let (sql, relations, events) = entry; - let entry = MacroCallEntry { - sql, - relations, - events: events - .into_iter() - .map(event_from_row) - .collect::>()?, - }; - self.inner - .lock() - .map_err(poisoned)? - .record(class_id, call_text, entry); + /// Note how many modules were loaded when the store was trusted, or `None` once it is not. + fn observe_modules(&self, count: Option) -> PyResult<()> { + *self.observed_modules.lock().map_err(poisoned)? = count; Ok(()) } @@ -93,15 +76,6 @@ impl MacroCallMemo { Ok(()) } - /// Store results of `class_id` under `class_text` and everything the store key covers. - fn set_persistent_class(&self, class_id: u64, class_text: &str) -> PyResult<()> { - self.inner - .lock() - .map_err(poisoned)? - .set_persistent_class(class_id, class_text); - Ok(()) - } - /// Calls found in the store and calls recorded into it. fn store_stats(&self) -> PyResult<(usize, usize)> { Ok(self.inner.lock().map_err(poisoned)?.store_stats()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs index bff5a278ed..33941bf858 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs @@ -1,23 +1,11 @@ -//! Python rows of macro call sites, splice spans and recorded call events. +//! Python rows of macro call sites and splice spans. -use pyo3::PyResult; -use pyo3::exceptions::PyValueError; -use sqlbuild_render::macro_calls::models::{MacroCallEvent, MacroCallSite, MacroSpliceSpan}; - -/// Event tags shared with `sqlbuild.compiler.macro_bridge.constants`. -const MACRO_USE: u8 = 0; -const DECLARATION_READ: u8 = 1; -const GENERATED_SQL: u8 = 2; -const ARGUMENT_REFERENCE: u8 = 3; +use sqlbuild_render::macro_calls::models::{MacroCallSite, MacroSpliceSpan}; /// One call site: start, arguments start, end, name, tree names if scanned and whether typed reference text appears. pub(crate) type SiteRow = (usize, usize, usize, String, Option>, bool); /// One substitution span: source start and end, output start and end. pub(crate) type SpanRow = (usize, usize, usize, usize); -/// One recorded event: its tag and two text fields. -pub(crate) type EventRow = (u8, String, String); -/// One recorded call: its SQL, added relations and events. -pub(crate) type EntryRow = (String, Vec<(String, String)>, Vec); pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { ( @@ -38,39 +26,3 @@ pub(crate) fn span_row(span: MacroSpliceSpan) -> SpanRow { span.output_end, ) } - -pub(crate) fn event_row(event: &MacroCallEvent) -> EventRow { - match event { - MacroCallEvent::MacroUse { name } => (MACRO_USE, name.clone(), String::new()), - MacroCallEvent::DeclarationRead { kind, name } => { - (DECLARATION_READ, kind.clone(), name.clone()) - } - MacroCallEvent::GeneratedSql { macro_name, sql } => { - (GENERATED_SQL, macro_name.clone(), sql.clone()) - } - MacroCallEvent::ArgumentReference { kind, name } => { - (ARGUMENT_REFERENCE, kind.clone(), name.clone()) - } - } -} - -pub(crate) fn event_from_row((tag, first, second): EventRow) -> PyResult { - match tag { - MACRO_USE => Ok(MacroCallEvent::MacroUse { name: first }), - DECLARATION_READ => Ok(MacroCallEvent::DeclarationRead { - kind: first, - name: second, - }), - GENERATED_SQL => Ok(MacroCallEvent::GeneratedSql { - macro_name: first, - sql: second, - }), - ARGUMENT_REFERENCE => Ok(MacroCallEvent::ArgumentReference { - kind: first, - name: second, - }), - _ => Err(PyValueError::new_err(format!( - "unknown macro call event tag {tag}" - ))), - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs new file mode 100644 index 0000000000..3e8246ab9c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs @@ -0,0 +1,430 @@ +//! One macro call evaluated natively: its nested calls and arguments, then the macro body in Python. + +use std::collections::HashSet; + +use pyo3::exceptions::{PyException, PyTypeError, PyUnicodeEncodeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple, PyTupleMethods, + PyTypeMethods, +}; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_render::macro_arguments::errors::ArgumentError; +use sqlbuild_render::macro_arguments::models::{ArgumentValue, MacroArguments}; +use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names as scan_names; +use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; +use sqlbuild_render::macro_calls::models::{MacroCallScan, MacroCallSite}; + +use crate::bindings::_helpers::render::macro_arguments::parsed_arguments; +use crate::bindings::_helpers::render::macro_expansion::{ + ExpansionClasses, NativeMacroExpansion, raised, +}; +use crate::bindings::_helpers::render::macro_expansion_facts::ResourceRef; + +const CONTEXT_PARAMETER: &str = "ctx"; +const SURROGATE_HELP: &str = "Return valid Unicode text from the macro; lone surrogates usually \ + come from bytes decoded with errors='surrogateescape'"; +const UNARY_HELP: &str = + "Return a number from the nested macro, or apply the sign inside the macro"; + +/// Text addressed by code point offsets, as the native scan reports them. +pub(crate) struct CallText<'a> { + pub(crate) text: &'a str, + /// The byte offset of each code point, then the text's length. + bytes: Vec, +} + +impl<'a> CallText<'a> { + pub(crate) fn new(text: &'a str) -> Self { + let mut bytes: Vec = text.char_indices().map(|(offset, _)| offset).collect(); + bytes.push(text.len()); + Self { text, bytes } + } + + /// The code points `start..end`. + pub(crate) fn slice(&self, start: usize, end: usize) -> &'a str { + &self.text[self.bytes[start]..self.bytes[end]] + } +} + +/// One expansion's state, its Python handle and the classes it builds objects from. +pub(crate) struct Evaluation<'a, 'py> { + pub(crate) expansion: &'a NativeMacroExpansion, + /// The expansion as the Python object macro contexts and the relation renderer call back into. + pub(crate) handle: &'a Bound<'py, PyAny>, + pub(crate) classes: &'a ExpansionClasses<'py>, +} + +/// `evaluate_macro_call`: one scanned call's value; a top-level call must return SQL. +pub(crate) fn evaluate_call<'py>( + run: &Evaluation<'_, 'py>, + call: (&CallText<'_>, &MacroCallSite), + top_level: bool, +) -> PyResult> { + let (text, site) = call; + let (expansion, classes) = (run.expansion, run.classes); + let py: Python<'py> = run.handle.py(); + let name: &str = &site.name; + let loaded_macro: Bound<'py, PyAny> = expansion.resolved_macro(classes, name, py)?; + if let Some(value) = expansion.overrides.bind(py).get_item(name)? + && !value.is_none() + { + let _ = final_sql(run, name, &value)?; + return Ok(value); + } + let identity: Bound<'py, PyAny> = expansion.macro_identity(classes, name, py)?; + expansion.record_macro_use(classes, name, &identity)?; + let args_source: &str = text.slice(site.args_start, site.end - 1); + expansion.facts.borrow_mut().open_call_site(); + let parsed: PyResult<(Bound<'py, PyTuple>, Bound<'py, PyDict>)> = + call_arguments(run, name, args_source); + let approved: HashSet = expansion.facts.borrow_mut().close_call_site(); + let arguments = match parsed { + Ok(arguments) => arguments, + Err(error) if error.is_instance(py, &classes.macro_argument_error) => { + let shifted: Bound<'py, PyAny> = error + .value(py) + .call_method1("shifted", (site.args_start,))?; + shifted.setattr("__suppress_context__", true)?; + return Err(PyErr::from_value(shifted)); + } + Err(error) => return Err(error), + }; + let result: Bound<'py, PyAny> = + called_macro(run, (name, &loaded_macro), arguments, approved) + .map_err(|error| macro_failure(classes, error, name, &expansion.file_label))?; + let Ok(sql) = result.downcast::() else { + if top_level { + let message: String = format!( + "Macro '@{name}' in '{}' must return a SQL string when used directly in SQL", + expansion.file_label + ); + return Err(raised(&classes.compile_error, &message)); + } + return Ok(result); + }; + let sql: String = final_sql(run, name, sql.as_any())?; + expansion.report_generated(classes, name, &loaded_macro, &sql)?; + Ok(result) +} + +/// The macro's result, called with its context, arguments and call-site references approved. +fn called_macro<'py>( + run: &Evaluation<'_, 'py>, + called: (&str, &Bound<'py, PyAny>), + arguments: (Bound<'py, PyTuple>, Bound<'py, PyDict>), + approved: HashSet, +) -> PyResult> { + let (name, loaded_macro) = called; + let (args, kwargs) = arguments; + let (expansion, classes) = (run.expansion, run.classes); + let py: Python<'py> = run.handle.py(); + let context_arguments: Bound<'py, PyDict> = PyDict::new(py); + context_arguments.set_item("expansion", run.handle)?; + context_arguments.set_item("declarations", expansion.declarations.bind(py))?; + context_arguments.set_item("file_path", expansion.file_path.bind(py))?; + context_arguments.set_item("macro_context", expansion.macro_context.bind(py))?; + let context: Bound<'py, PyAny> = classes + .invocation_context + .call((), Some(&context_arguments))?; + expansion.facts.borrow_mut().begin_rendering(approved); + let token: PyResult> = run + .handle + .getattr("render_relation") + .and_then(|renderer| classes.relation_renderer.call_method1("set", (renderer,))); + let result: PyResult> = token + .as_ref() + .map_err(|error| error.clone_ref(py)) + .and_then(|_| invoke(classes, (name, loaded_macro), &context, (&args, &kwargs))); + let reset: PyResult<()> = match &token { + Ok(token) => classes + .relation_renderer + .call_method1("reset", (token,)) + .map(|_| ()), + Err(_) => Ok(()), + }; + expansion.facts.borrow_mut().end_rendering(); + let result: Bound<'py, PyAny> = result?; + reset?; + Ok(result) +} + +/// `_call_loaded_macro`: the macro called with its injected context when it takes one. +fn invoke<'py>( + classes: &ExpansionClasses<'py>, + called: (&str, &Bound<'py, PyAny>), + context: &Bound<'py, PyAny>, + arguments: (&Bound<'py, PyTuple>, &Bound<'py, PyDict>), +) -> PyResult> { + let (name, loaded_macro) = called; + let (args, kwargs) = arguments; + let function: Bound<'py, PyAny> = loaded_macro.getattr("function")?; + if !loaded_macro.getattr("injects_context")?.is_truthy()? { + return function.call(args, Some(kwargs)); + } + if kwargs.contains(CONTEXT_PARAMETER)? { + let message: String = format!( + "Macro '@{name}' must not be called with keyword argument 'ctx'; 'ctx' is reserved \ + for injected macro context" + ); + return Err(raised(&classes.compile_error, &message)); + } + let mut positional: Vec> = vec![context.clone()]; + positional.extend(args.iter()); + function.call(PyTuple::new(args.py(), positional)?, Some(kwargs)) +} + +/// Python's `except` clauses around a macro call: compile errors pass, other exceptions wrap. +fn macro_failure( + classes: &ExpansionClasses<'_>, + error: PyErr, + name: &str, + file_label: &str, +) -> PyErr { + let py: Python<'_> = classes.compile_error.py(); + if error.is_instance(py, &classes.compile_error) { + return error; + } + let reason: &str = if error.is_instance_of::(py) { + "could not be called" + } else if error.is_instance_of::(py) { + "failed" + } else { + return error; + }; + let detail: String = match error.value(py).str() { + Ok(text) => text.to_string(), + Err(failure) => return failure, + }; + let wrapped: PyErr = raised( + &classes.compile_error, + &format!("Macro '@{name}' in '{file_label}' {reason}: {detail}"), + ); + wrapped.set_cause(py, Some(error)); + wrapped +} + +/// `_validate_final_macro_sql`: valid Unicode without SQLBuild calls, as Rust text. +fn final_sql(run: &Evaluation<'_, '_>, name: &str, value: &Bound<'_, PyAny>) -> PyResult { + let (expansion, classes) = (run.expansion, run.classes); + let py: Python<'_> = value.py(); + if let Err(error) = value.call_method1("encode", ("utf-8",)) { + if !error.is_instance_of::(py) { + return Err(error); + } + let start: Bound<'_, PyAny> = error.value(py).getattr("start")?; + let surrogate: String = value.get_item(start)?.repr()?.to_string(); + let message: String = format!( + "Macro '@{name}' in '{}' returned text with the lone surrogate {surrogate}, which is \ + not valid Unicode", + expansion.file_label + ); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", SURROGATE_HELP)?; + let raised_error: Bound<'_, PyAny> = + classes.compile_error.call((message,), Some(&keywords))?; + raised_error.setattr("__suppress_context__", true)?; + return Err(PyErr::from_value(raised_error)); + } + let sql: String = value.extract()?; + let names: Vec = scan_names(expansion.python, &sql, false) + .map_err(|error| raised(&classes.compile_error, error.message()))?; + if names.is_empty() { + return Ok(sql); + } + let calls: String = names + .iter() + .map(|call| format!("@{call}()")) + .collect::>() + .join(", "); + let message: String = format!( + "Macro '@{name}' in '{}' returned SQLBuild call(s) {calls}. Macro output must be final \ + SQL. Compose macros with ordinary Python imports and function calls instead; nested \ + @macro() calls are supported only when explicitly authored in SQL", + expansion.file_label + ); + Err(raised(&classes.compile_error, &message)) +} + +/// `_parse_macro_arguments`: nested calls evaluated in order, then the arguments built. +fn call_arguments<'py>( + run: &Evaluation<'_, 'py>, + name: &str, + args_source: &str, +) -> PyResult<(Bound<'py, PyTuple>, Bound<'py, PyDict>)> { + let py: Python<'py> = run.handle.py(); + if python_strip(args_source).is_empty() { + return Ok((PyTuple::empty(py), PyDict::new(py))); + } + let text: CallText<'_> = CallText::new(args_source); + let scan: MacroCallScan = scan_sites(run.expansion.python, args_source); + let mut values: Vec> = Vec::with_capacity(scan.sites.len()); + for site in &scan.sites { + values.push(evaluate_call(run, (&text, site), false)?); + } + if let Some(failure) = &scan.failure { + return Err(run.expansion.failed_scan(run.classes, failure, py)); + } + let nested: Vec<(usize, usize)> = scan + .sites + .iter() + .map(|site| (site.start, site.end)) + .collect(); + let source: ArgumentSource<'_, 'py> = ArgumentSource { + run, + name, + text: &text, + nested: &nested, + values: &values, + }; + let arguments: MacroArguments = match parsed_arguments(py, args_source, &nested)? { + Ok(arguments) => arguments, + Err(error) => return Err(source.parse_error(&error)), + }; + let mut positional: Vec> = Vec::with_capacity(arguments.positional.len()); + for value in &arguments.positional { + positional.push(source.built(value)?); + } + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (keyword, value) in &arguments.keywords { + keywords.set_item(keyword, source.built(value)?)?; + } + let mut references: Vec = Vec::with_capacity(arguments.typed_references.len()); + for (function, reference_name) in &arguments.typed_references { + let kind: String = run + .classes + .ref_kinds + .get_item(function)? + .ok_or_else(|| PyValueError::new_err(function.clone()))? + .getattr("value")? + .extract()?; + references.push((kind, reference_name.clone())); + } + run.expansion + .facts + .borrow_mut() + .record_call_site_refs(&references); + Ok((PyTuple::new(py, positional)?, keywords)) +} + +/// One call's argument text, its nested call spans and their values. +struct ArgumentSource<'a, 'py> { + run: &'a Evaluation<'a, 'py>, + name: &'a str, + text: &'a CallText<'a>, + nested: &'a [(usize, usize)], + values: &'a [Bound<'py, PyAny>], +} + +impl<'py> ArgumentSource<'_, 'py> { + /// `_built`: the Python object one planned value stands for. + fn built(&self, value: &ArgumentValue) -> PyResult> { + let classes: &ExpansionClasses<'py> = self.run.classes; + let py: Python<'py> = self.run.handle.py(); + Ok(match value { + ArgumentValue::Str(text) => PyString::new(py, text).into_any(), + ArgumentValue::Bool(flag) => PyBool::new(py, *flag).to_owned().into_any(), + ArgumentValue::None => py.None().into_bound(py), + ArgumentValue::Int { radix, digits } => classes.int_type.call1((digits, radix))?, + ArgumentValue::Float(text) => classes.float_type.call1((text,))?, + ArgumentValue::NestedCall(index) => self.values[*index].clone(), + ArgumentValue::TypedReference { function, name } => { + let kind: Bound<'py, PyAny> = classes + .ref_kinds + .get_item(function)? + .ok_or_else(|| PyValueError::new_err(function.clone()))?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("kind", kind)?; + fields.set_item("name", name)?; + classes.resource_ref.call((), Some(&fields))? + } + ArgumentValue::List(items) => PyList::new(py, self.items(items)?)?.into_any(), + ArgumentValue::Tuple(items) => PyTuple::new(py, self.items(items)?)?.into_any(), + ArgumentValue::Dict(pairs) => { + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for (key, item) in pairs { + let key: Bound<'py, PyAny> = self.built(key)?; + mapping.set_item(key, self.built(item)?)?; + } + mapping.into_any() + } + ArgumentValue::Negative(inner) => self.number(inner, "-")?.neg()?, + ArgumentValue::Positive(inner) => self.number(inner, "+")?.pos()?, + }) + } + + fn items(&self, items: &[ArgumentValue]) -> PyResult>> { + items.iter().map(|item| self.built(item)).collect() + } + + /// `_number`: the operand of a unary sign, which a nested call must return as a number. + fn number(&self, operand: &ArgumentValue, sign: &str) -> PyResult> { + let number: Bound<'py, PyAny> = self.built(operand)?; + if number.is_instance_of::() || number.is_instance_of::() { + return Ok(number); + } + let mut row: &ArgumentValue = operand; + let call: usize = loop { + match row { + ArgumentValue::NestedCall(index) => break *index, + ArgumentValue::Negative(inner) | ArgumentValue::Positive(inner) => row = inner, + _ => { + return Err(PyValueError::new_err( + "unary sign over a non-number literal", + )); + } + } + }; + let type_name: String = self.values[call].get_type().name()?.to_string(); + let detail: String = format!( + "apply unary {sign} to the value of a nested macro call, which is a {type_name} and \ + not a number" + ); + Err(self.argument_error(self.nested[call].0, &detail, UNARY_HELP)) + } + + /// The `MacroArgumentError` for a native parse error at its line and column. + fn parse_error(&self, error: &ArgumentError) -> PyErr { + let mut offset: usize = 0; + for line in self + .text + .text + .split('\n') + .take(error.line.saturating_sub(1)) + { + offset += line.chars().count() + 1; + } + offset += error.column.saturating_sub(1); + self.argument_error(offset, &error.detail, &error.help) + } + + /// `MacroArgumentError` at a code point offset of the argument text. + fn argument_error(&self, offset: usize, detail: &str, help: &str) -> PyErr { + let before: &str = self.text.slice(0, offset.min(self.text.bytes.len() - 1)); + let line: usize = before.matches('\n').count() + 1; + let column: usize = before + .rsplit_once('\n') + .map_or(before, |(_, last)| last) + .chars() + .count() + + 1; + let created = (|| { + let py: Python<'_> = self.run.handle.py(); + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("detail", detail)?; + fields.set_item("help", help)?; + fields.set_item("macro_name", self.name)?; + fields.set_item("file_path", self.run.expansion.file_path.bind(py))?; + fields.set_item("offset", offset)?; + fields.set_item("relative_position", (line, column))?; + self.run + .classes + .macro_argument_error + .call((), Some(&fields)) + })(); + match created { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs new file mode 100644 index 0000000000..c31cd6bc17 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs @@ -0,0 +1,783 @@ +//! One authored string's macro expansion: scanned, memoised, evaluated and spliced natively. + +use std::cell::RefCell; + +use pyo3::exceptions::{PyIndexError, PyValueError}; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTuple}; +use pyo3::{FromPyObject, pyclass, pymethods}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::is_python_decimal::is_python_decimal; +use sqlbuild_core::text::models::PythonText; +use sqlbuild_render::macro_calls::main::scan_macro_call_sites::scan_macro_call_sites as scan_sites; +use sqlbuild_render::macro_calls::main::splice_macro_calls::splice_macro_calls as splice_calls; +use sqlbuild_render::macro_calls::models::{ + MacroCallEntry, MacroCallEvent, MacroCallScan, MacroCallSite, MacroScanFailure, MacroSpliceSpan, +}; + +use crate::bindings::_helpers::render::declaration_expansion::usage_visibility; +use crate::bindings::_helpers::render::macro_call_classes::{ + CallClass, ClassInputs, StoreTokenClasses, +}; +use crate::bindings::_helpers::render::macro_call_memo::{MacroCallMemo, poisoned}; +use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; +use crate::bindings::_helpers::render::macro_evaluation::{CallText, Evaluation, evaluate_call}; +use crate::bindings::_helpers::render::macro_expansion_facts::{ExpansionFacts, ResourceRef}; +use crate::bindings::_helpers::render::macro_generated_references::{ + GeneratedOrigin, GeneratedReferenceClasses, report_generated_references, +}; + +const RELATION_PREFIX: &str = "__sqlbuild_relation_"; +const MEMO_ATTRIBUTE: &str = "memo"; +const OBSERVE_MODULES_METHOD: &str = "observe_modules"; +const RELATION_SUFFIX: &str = "__"; +const CONSTANT_KIND: &str = "constant"; +const RELATIONSHIP_CONSUMER_KINDS: [&str; 2] = ["test", "scenario"]; + +/// `(sql, spans, argument references, dependencies, usages)` of one expansion. +type ExpansionRow<'py> = ( + String, + Vec, + Vec, + Vec>, + Vec>, +); + +/// The Python classes, enums and helpers an expansion builds facts, errors and contexts from. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct ExpansionClasses<'py> { + pub(crate) compile_error: Bound<'py, PyAny>, + pub(crate) macro_argument_error: Bound<'py, PyAny>, + mismatch_error: Bound<'py, PyAny>, + declaration_identity: Bound<'py, PyAny>, + declaration_kind: Bound<'py, PyAny>, + macro_kind: Bound<'py, PyAny>, + usage_record: Bound<'py, PyAny>, + runtime_usage: Bound<'py, PyAny>, + resource_identity: Bound<'py, PyAny>, + model_kind: Bound<'py, PyAny>, + pub(crate) resource_ref: Bound<'py, PyAny>, + /// `SqlResourceRefKind` member by typed reference function, such as `__ref`. + pub(crate) ref_kinds: Bound<'py, PyDict>, + /// Typed reference function by `SqlResourceRefKind` value. + ref_functions: Bound<'py, PyDict>, + /// `macro_store_token` and `context_store_token`. + store_tokens: StoreTokenClasses<'py>, + pub(crate) relation_renderer: Bound<'py, PyAny>, + /// Builds the `MacroContext` a macro is called with from the visible declarations. + pub(crate) invocation_context: Bound<'py, PyAny>, + /// What reference calls a macro returned are reported with. + generated: GeneratedReferenceClasses<'py>, + /// Counts a call run outside the memo because a test mocks its macro. + report_mocked: Bound<'py, PyAny>, + pub(crate) int_type: Bound<'py, PyAny>, + pub(crate) float_type: Bound<'py, PyAny>, +} + +/// One authored string's expansion inputs and the facts it records. +#[pyclass(module = "sqlbuild._native", unsendable)] +pub(crate) struct NativeMacroExpansion { + pub(crate) file_path: Py, + pub(crate) file_label: String, + pub(crate) loaded_macros: Py, + pub(crate) overrides: Py, + pub(crate) macro_context: Py, + /// The visible `DeclarationResolutionContext`, or `None`. + pub(crate) declarations: Py, + /// The consumer usages are recorded for, or `None`. + pub(crate) consumer: Py, + enforce_explicit: bool, + classes: Py, + pub(crate) python: PythonText, + pub(crate) facts: RefCell, +} + +/// The `(macros, overrides, context, declarations, consumer)` one expansion reads. +type StateInput = (Py, Py, Py, Py, Py); + +#[pymethods] +impl NativeMacroExpansion { + #[new] + pub(crate) fn new( + file_path: &Bound<'_, PyAny>, + state: StateInput, + enforce_explicit: bool, + classes: Py, + ) -> PyResult { + let (loaded_macros, overrides, macro_context, declarations, consumer) = state; + Ok(Self { + file_path: file_path.clone().unbind(), + file_label: file_path.str()?.to_string(), + loaded_macros, + overrides, + macro_context, + declarations, + consumer, + enforce_explicit, + classes, + python: active_python_text(), + facts: RefCell::new(ExpansionFacts::default()), + }) + } + + /// Expand `sql`, through the bridge's memo when one is active. + fn expand<'py>( + slf: &Bound<'py, Self>, + sql: &Bound<'py, PyString>, + bridge: &Bound<'py, PyAny>, + ) -> PyResult> { + let output: ExpansionOutput = expanded(slf, sql.to_str()?, bridge)?; + Ok(( + output.sql, + output.spans.into_iter().map(span_row).collect(), + output.argument_references, + output.dependencies, + output.usages, + )) + } + + /// `render_relation`: an approved call-site reference's placeholder, else its reference call. + fn render_relation(&self, reference: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = reference.py(); + let classes: ExpansionClasses<'_> = self.classes.bind(py).extract()?; + if !reference.is_instance(&classes.resource_ref)? { + return Err(raised( + &classes.compile_error, + "Only typed resource references render as relations", + )); + } + let key: ResourceRef = resource_key(reference)?; + let index: Option = self.facts.borrow_mut().relation_index(&key); + match index { + Some(index) => Ok(format!("{RELATION_PREFIX}{index}{RELATION_SUFFIX}")), + None => reference_call_text(&classes, &key), + } + } + + /// `_record_macro_declaration_usage`: a macro read a constant or enum through its context. + fn declaration_read(&self, kind: &str, name: &str, py: Python<'_>) -> PyResult<()> { + let classes: ExpansionClasses<'_> = self.classes.bind(py).extract()?; + self.record_declaration_read(&classes, kind, name) + } + + /// Mark the executing call as dependent on its consumer, so it is never replayed. + fn block_memo(&self) { + self.facts.borrow_mut().block_memo(); + } +} + +impl NativeMacroExpansion { + /// Each site's call class, or `None` when the scan failed or a tree name is unresolvable. + fn call_classes<'py>( + &self, + classes: &ExpansionClasses<'py>, + memo: &MacroCallMemo, + scan: &MacroCallScan, + ) -> PyResult>> { + if scan.failure.is_some() { + return Ok(None); + } + let py: Python<'py> = classes.compile_error.py(); + let inputs: ClassInputs<'_, 'py> = ClassInputs { + declarations: self.declarations.bind(py), + loaded_macros: self.loaded_macros.bind(py), + macro_context: self.macro_context.bind(py), + store: &classes.store_tokens, + }; + let mut state = memo.classes.lock().map_err(poisoned)?; + let mut found: Vec = Vec::with_capacity(scan.sites.len()); + for site in &scan.sites { + let Some(tree_names) = &site.tree_names else { + return Ok(None); + }; + let Some(call_class) = state.call_class(tree_names, &inputs)? else { + return Ok(None); + }; + found.push(call_class); + } + Ok(Some(found)) + } + + /// `_bridged_call_output`: a recorded call replayed, or the call evaluated and recorded. + fn bridged_output<'py>( + &self, + run: &Evaluation<'_, 'py>, + bridge: (&Bound<'py, PyAny>, &MacroCallMemo), + call: (&CallText<'_>, &MacroCallSite), + call_class: &CallClass, + ) -> PyResult { + let (bridge, memo) = bridge; + let (text, site) = call; + let classes: &ExpansionClasses<'py> = run.classes; + let py: Python<'py> = bridge.py(); + if self.mocked(site, py)? { + let _ = classes.report_mocked.call0()?; + return self.evaluated_output(run, (text, site)); + } + let call_text: &str = text.slice(site.start, site.end); + let prior: Option> = site + .typed_reference_text + .then(|| self.facts.borrow().relations.clone()); + let (class_id, class_text) = memo + .classes + .lock() + .map_err(poisoned)? + .class_id(call_class, prior.as_deref()); + let recorded: Option = { + let mut inner = memo.inner.lock().map_err(poisoned)?; + if let Some(class_text) = class_text { + inner.set_persistent_class(class_id, &class_text); + } + inner + .lookup(class_id, call_text) + .map(|entry| MacroCallEntry::clone(&entry)) + }; + if let Some(entry) = recorded { + for event in entry.events { + self.replay(classes, event)?; + } + for relation in &entry.relations { + let _ = self.facts.borrow_mut().relation_slot(relation); + } + return self.render_placeholders(classes, &entry.sql); + } + let known: usize = self.facts.borrow().relations.len(); + self.facts.borrow_mut().begin_event_log(); + let result: PyResult = top_level_output(run, (text, site)); + let (events, replayable) = self.facts.borrow_mut().end_event_log(); + let output: String = result?; + let added: Vec = self.facts.borrow().relations[known..].to_vec(); + if replayable && (prior.is_some() || added.is_empty()) { + observe_new_modules(bridge, memo)?; + let entry: MacroCallEntry = MacroCallEntry { + sql: output.clone(), + relations: added, + events, + }; + memo.inner + .lock() + .map_err(poisoned)? + .record(class_id, call_text.to_owned(), entry); + } + self.render_placeholders(classes, &output) + } + + /// `_unmemoised_sql_macros`: every call evaluated in order, then the scan's failure raised. + fn unmemoised_outputs( + &self, + run: &Evaluation<'_, '_>, + text: &CallText<'_>, + scan: &MacroCallScan, + ) -> PyResult> { + let mut outputs: Vec = Vec::with_capacity(scan.sites.len()); + for site in &scan.sites { + outputs.push(self.evaluated_output(run, (text, site))?); + } + if let Some(failure) = &scan.failure { + return Err(self.failed_scan(run.classes, failure, run.handle.py())); + } + Ok(outputs) + } + + fn evaluated_output( + &self, + run: &Evaluation<'_, '_>, + call: (&CallText<'_>, &MacroCallSite), + ) -> PyResult { + let output: String = top_level_output(run, call)?; + self.render_placeholders(run.classes, &output) + } + + /// Whether a test overrides any macro in the call's tree. + fn mocked(&self, site: &MacroCallSite, py: Python<'_>) -> PyResult { + let overrides: &Bound<'_, PyDict> = self.overrides.bind(py); + for name in site.tree_names.iter().flatten() { + if overrides.contains(name)? { + return Ok(true); + } + } + Ok(false) + } + + /// `raise_failed_macro_call`: the failing call's macro lookup error, else the scan's error. + pub(crate) fn failed_scan( + &self, + classes: &ExpansionClasses<'_>, + failure: &MacroScanFailure, + py: Python<'_>, + ) -> PyErr { + if let Some(name) = &failure.call_name + && let Err(error) = self.resolved_macro(classes, name, py) + { + return error; + } + raised(&classes.compile_error, failure.error.message()) + } + + /// `_resolved_macro`: the visible macro `name`, or Python's inaccessible or unknown error. + pub(crate) fn resolved_macro<'py>( + &self, + classes: &ExpansionClasses<'py>, + name: &str, + py: Python<'py>, + ) -> PyResult> { + let declarations: &Bound<'py, PyAny> = self.declarations.bind(py); + let macros: Bound<'py, PyAny> = if declarations.is_none() { + self.loaded_macros.bind(py).clone().into_any() + } else { + declarations.getattr("macros")? + }; + if !declarations.is_none() { + let inaccessible: Bound<'py, PyAny> = declarations.getattr("inaccessible_macros")?; + if inaccessible.contains(name)? { + let record: Bound<'py, PyAny> = inaccessible.get_item(name)?; + let owning: Bound<'py, PyAny> = record.getattr("owning_path")?; + let owner: String = if owning.is_truthy()? { + owning.str()?.to_string() + } else { + "global".to_owned() + }; + let message: String = format!( + "Macro '@{name}' in '{}' is inaccessible. Defined at '{}:{}:{}' with scope \ + owner '{owner}'; consumer/definition path is '{}'. Visible macros: {}", + self.file_label, + record.getattr("path")?.str()?, + record.getattr("line")?.str()?, + record.getattr("column")?.str()?, + self.file_label, + sorted_names(¯os)?, + ); + return Err(raised(&classes.compile_error, &message)); + } + } + if macros.contains(name)? { + return macros.get_item(name); + } + let message: String = format!( + "Unknown macro '@{name}' in '{}'. Available macros: {}", + self.file_label, + sorted_names(¯os)? + ); + Err(raised(&classes.compile_error, &message)) + } + + /// `_record_macro_use`: the macro's dependency and its runtime usage by the consumer. + pub(crate) fn record_macro_use<'py>( + &self, + classes: &ExpansionClasses<'py>, + name: &str, + identity: &Bound<'py, PyAny>, + ) -> PyResult<()> { + let py: Python<'py> = identity.py(); + { + let mut facts = self.facts.borrow_mut(); + facts.note_event(MacroCallEvent::MacroUse { + name: name.to_owned(), + }); + facts.dependencies.push(identity.clone().unbind()); + } + let consumer: &Bound<'py, PyAny> = self.consumer.bind(py); + if consumer.is_none() { + return Ok(()); + } + let declarations: &Bound<'py, PyAny> = self.declarations.bind(py); + let visibility: Bound<'py, PyAny> = if declarations.is_none() { + PyTuple::empty(py).into_any() + } else { + declarations + .getattr("macro_visibility")? + .call_method1("get", (name, PyTuple::empty(py)))? + }; + let usages: Vec> = if visibility.is_truthy()? { + let mut usages: Vec> = Vec::new(); + for visible in usage_visibility(&visibility, relationship_consumer(classes, consumer)?)? + { + usages.push(usage( + classes, + (consumer, identity), + Some(&visible.getattr("through")?), + )?); + } + usages + } else { + vec![usage(classes, (consumer, identity), None)?] + }; + self.facts + .borrow_mut() + .usages + .extend(usages.into_iter().map(Bound::unbind)); + Ok(()) + } + + /// `_record_macro_declaration_usage` for a constant or enum the macro read. + fn record_declaration_read( + &self, + classes: &ExpansionClasses<'_>, + kind: &str, + name: &str, + ) -> PyResult<()> { + let py: Python<'_> = classes.compile_error.py(); + self.facts + .borrow_mut() + .note_event(MacroCallEvent::DeclarationRead { + kind: kind.to_owned(), + name: name.to_owned(), + }); + let declarations: &Bound<'_, PyAny> = self.declarations.bind(py); + let own_consumer: &Bound<'_, PyAny> = self.consumer.bind(py); + let consumer: Bound<'_, PyAny> = if own_consumer.is_truthy()? { + own_consumer.clone() + } else { + declarations.getattr("consumer")? + }; + if consumer.is_none() { + return Ok(()); + } + let constant: bool = kind == CONSTANT_KIND; + let visibility: Bound<'_, PyAny> = declarations + .getattr(if constant { + "constant_visibility" + } else { + "enum_visibility" + })? + .call_method1("get", (name, PyTuple::empty(py)))?; + let mut usages: Vec> = Vec::new(); + if visibility.is_truthy()? { + for visible in + usage_visibility(&visibility, relationship_consumer(classes, &consumer)?)? + { + usages.push(usage( + classes, + (&consumer, &visible.getattr("declaration")?), + Some(&visible.getattr("through")?), + )?); + } + } else { + let declaration: Bound<'_, PyAny> = declarations + .getattr(if constant { "constants" } else { "enums" })? + .get_item(name)?; + let model_name: Bound<'_, PyAny> = declaration.getattr("model_name")?; + let owner: Option> = if model_name.is_none() { + None + } else { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("kind", &classes.model_kind)?; + fields.set_item("name", model_name)?; + Some(classes.resource_identity.call((), Some(&fields))?) + }; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("kind", classes.declaration_kind.call1((kind,))?)?; + fields.set_item("name", name)?; + fields.set_item("owner", owner)?; + let identity: Bound<'_, PyAny> = + classes.declaration_identity.call((), Some(&fields))?; + usages.push(usage(classes, (&consumer, &identity), None)?); + } + self.facts + .borrow_mut() + .usages + .extend(usages.into_iter().map(Bound::unbind)); + Ok(()) + } + + /// `_report_generated_references` for a macro's returned SQL. + pub(crate) fn report_generated( + &self, + classes: &ExpansionClasses<'_>, + name: &str, + loaded_macro: &Bound<'_, PyAny>, + result: &str, + ) -> PyResult<()> { + if result_has_markers(classes, result) { + self.facts + .borrow_mut() + .note_event(MacroCallEvent::GeneratedSql { + macro_name: name.to_owned(), + sql: result.to_owned(), + }); + } + let py: Python<'_> = loaded_macro.py(); + let origin: GeneratedOrigin<'_, '_> = GeneratedOrigin { + file_path: self.file_path.bind(py), + consumer: self.consumer.bind(py), + enforce_explicit: self.enforce_explicit, + }; + report_generated_references(&classes.generated, &origin, loaded_macro, result) + } + + /// `_replay_macro_call_event`: one recorded fact, recorded again for this consumer. + fn replay(&self, classes: &ExpansionClasses<'_>, event: MacroCallEvent) -> PyResult<()> { + let py: Python<'_> = classes.compile_error.py(); + match event { + MacroCallEvent::MacroUse { name } => { + let identity: Bound<'_, PyAny> = self.macro_identity(classes, &name, py)?; + self.record_macro_use(classes, &name, &identity) + } + MacroCallEvent::DeclarationRead { kind, name } => { + if self.declarations.bind(py).is_none() { + return Err(raised( + &classes.mismatch_error, + "Replayed a declaration read without declarations", + )); + } + self.record_declaration_read(classes, &kind, &name) + } + MacroCallEvent::GeneratedSql { macro_name, sql } => { + let declarations: &Bound<'_, PyAny> = self.declarations.bind(py); + let loaded_macro: Bound<'_, PyAny> = if declarations.is_none() { + self.loaded_macros + .bind(py) + .get_item(¯o_name)? + .ok_or_else(|| { + PyErr::new::(macro_name.clone()) + })? + } else { + declarations.getattr("macros")?.get_item(¯o_name)? + }; + self.report_generated(classes, ¯o_name, &loaded_macro, &sql) + } + MacroCallEvent::ArgumentReference { kind, name } => { + self.facts + .borrow_mut() + .record_call_site_refs(&[(kind, name)]); + Ok(()) + } + } + } + + /// The macro's declaration identity: its indexed record's, else an unowned macro identity. + pub(crate) fn macro_identity<'py>( + &self, + classes: &ExpansionClasses<'py>, + name: &str, + py: Python<'py>, + ) -> PyResult> { + let declarations: &Bound<'py, PyAny> = self.declarations.bind(py); + if !declarations.is_none() { + return declarations + .getattr("macro_records")? + .get_item(name)? + .getattr("identity"); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("kind", &classes.macro_kind)?; + fields.set_item("name", name)?; + classes.declaration_identity.call((), Some(&fields)) + } + + /// `render_relation_placeholders`: each placeholder as the reference call it stands for. + fn render_placeholders(&self, classes: &ExpansionClasses<'_>, sql: &str) -> PyResult { + let relations: Vec = self.facts.borrow().relations.clone(); + if relations.is_empty() || !sql.contains(RELATION_PREFIX) { + return Ok(sql.to_owned()); + } + let mut rendered: String = String::with_capacity(sql.len()); + let mut cursor: usize = 0; + let mut search: usize = 0; + while let Some(found) = sql[search..].find(RELATION_PREFIX) { + let start: usize = search + found; + let digits_start: usize = start + RELATION_PREFIX.len(); + let digits_end: usize = sql[digits_start..] + .char_indices() + .find(|(_, character)| !is_python_decimal(self.python, *character)) + .map_or(sql.len(), |(offset, _)| digits_start + offset); + if digits_end == digits_start || !sql[digits_end..].starts_with(RELATION_SUFFIX) { + search = start + 1; + continue; + } + let index: usize = classes + .int_type + .call1((&sql[digits_start..digits_end],))? + .extract()?; + let relation: &ResourceRef = relations + .get(index) + .ok_or_else(|| PyIndexError::new_err("tuple index out of range"))?; + rendered.push_str(&sql[cursor..start]); + rendered.push_str(&reference_call_text(classes, relation)?); + cursor = digits_end + RELATION_SUFFIX.len(); + search = cursor; + } + rendered.push_str(&sql[cursor..]); + Ok(rendered) + } +} + +/// One expansion's SQL, substitution spans, call-site references, dependencies and usages. +pub(crate) struct ExpansionOutput { + pub(crate) sql: String, + pub(crate) spans: Vec, + pub(crate) argument_references: Vec, + pub(crate) dependencies: Vec>, + pub(crate) usages: Vec>, +} + +/// Expand `sql` with the expansion's inputs, through the bridge's memo when one is active. +pub(crate) fn expanded<'py>( + expansion: &Bound<'py, NativeMacroExpansion>, + sql: &str, + bridge: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = expansion.py(); + let this: PyRef<'py, NativeMacroExpansion> = expansion.borrow(); + let classes: ExpansionClasses<'py> = this.classes.bind(py).extract()?; + let handle: Bound<'py, PyAny> = expansion.as_any().clone(); + let run: Evaluation<'_, 'py> = Evaluation { + expansion: &this, + handle: &handle, + classes: &classes, + }; + let text: CallText<'_> = CallText::new(sql); + let scan: MacroCallScan = scan_sites(this.python, text.text); + let memo: Option> = if bridge.is_none() { + None + } else { + Some(bridge.getattr(MEMO_ATTRIBUTE)?.downcast_into()?) + }; + let call_classes: Option> = match &memo { + Some(memo) => this.call_classes(&classes, memo.get(), &scan)?, + None => None, + }; + let outputs: Vec = match (call_classes, &memo) { + (Some(call_classes), Some(memo)) => { + let mut outputs: Vec = Vec::with_capacity(scan.sites.len()); + for (site, call_class) in scan.sites.iter().zip(call_classes) { + outputs.push(this.bridged_output( + &run, + (bridge, memo.get()), + (&text, site), + &call_class, + )?); + } + outputs + } + _ => this.unmemoised_outputs(&run, &text, &scan)?, + }; + let bounds: Vec<(usize, usize)> = scan + .sites + .iter() + .map(|site| (site.start, site.end)) + .collect(); + let (rendered, spans) = splice_calls(text.text, &bounds, &outputs) + .ok_or_else(|| PyValueError::new_err("macro call sites do not fit the SQL"))?; + let facts = this.facts.borrow(); + Ok(ExpansionOutput { + sql: rendered, + spans, + argument_references: facts.argument_references.clone(), + dependencies: facts + .dependencies + .iter() + .map(|item| item.clone_ref(py)) + .collect(), + usages: facts.usages.iter().map(|item| item.clone_ref(py)).collect(), + }) +} + +/// One top-level call's SQL. +fn top_level_output( + run: &Evaluation<'_, '_>, + call: (&CallText<'_>, &MacroCallSite), +) -> PyResult { + evaluate_call(run, call, true)?.extract() +} + +/// A `SqlResourceRef`'s kind value and name. +pub(crate) fn resource_key(reference: &Bound<'_, PyAny>) -> PyResult { + Ok(( + reference.getattr("kind")?.getattr("value")?.extract()?, + reference.getattr("name")?.extract()?, + )) +} + +/// `reference_call_text`: the reference call naming `reference`, such as `__ref("orders")`. +fn reference_call_text( + classes: &ExpansionClasses<'_>, + reference: &ResourceRef, +) -> PyResult { + let (kind, name) = reference; + let function: String = classes + .ref_functions + .get_item(kind)? + .ok_or_else(|| PyErr::new::(kind.clone()))? + .extract()?; + Ok(format!("{function}(\"{name}\")")) +} + +/// Whether the consumer is a test or scenario, which records relationship visibility. +fn relationship_consumer( + classes: &ExpansionClasses<'_>, + consumer: &Bound<'_, PyAny>, +) -> PyResult { + if !consumer.is_instance(&classes.resource_identity)? { + return Ok(false); + } + let kind: String = consumer.getattr("kind")?.getattr("value")?.extract()?; + Ok(RELATIONSHIP_CONSUMER_KINDS.contains(&kind.as_str())) +} + +/// A runtime `UsageRecord` of `declaration` by `consumer`, through a relationship if any. +fn usage<'py>( + classes: &ExpansionClasses<'py>, + usage: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + through: Option<&Bound<'py, PyAny>>, +) -> PyResult> { + let (consumer, declaration) = usage; + let keywords: Bound<'py, PyDict> = PyDict::new(consumer.py()); + if let Some(through) = through { + keywords.set_item("through", through)?; + } + classes.usage_record.call( + (consumer, declaration, &classes.runtime_usage), + Some(&keywords), + ) +} + +/// `", ".join(sorted(names)) or "none"`. +fn sorted_names(names: &Bound<'_, PyAny>) -> PyResult { + let mut sorted: Vec = + PyList::new(names.py(), names.try_iter()?.collect::>>()?)?.extract()?; + sorted.sort_unstable(); + Ok(if sorted.is_empty() { + "none".to_owned() + } else { + sorted.join(", ") + }) +} + +/// The exception `class(message)` builds. +pub(crate) fn raised(class: &Bound<'_, PyAny>, message: &str) -> PyErr { + match class.call1((message,)) { + Ok(instance) => PyErr::from_value(instance), + Err(error) => error, + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} + +/// Whether a macro's result calls any reference function. +fn result_has_markers(classes: &ExpansionClasses<'_>, result: &str) -> bool { + classes + .generated + .markers() + .any(|marker| result.contains(marker)) +} + +/// Observe modules a recorded call imported, so the store is not saved over changed code. +fn observe_new_modules(bridge: &Bound<'_, PyAny>, memo: &MacroCallMemo) -> PyResult<()> { + let Some(observed) = *memo.observed_modules.lock().map_err(poisoned)? else { + return Ok(()); + }; + let loaded: usize = PyModule::import(bridge.py(), "sys")? + .getattr("modules")? + .len()?; + if loaded != observed { + let _ = bridge.call_method0(OBSERVE_MODULES_METHOD)?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs new file mode 100644 index 0000000000..0748b6edf3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs @@ -0,0 +1,110 @@ +//! Facts one authored string's macro expansion records, in encounter order. + +use std::collections::HashSet; + +use pyo3::prelude::{Py, PyAny}; +use sqlbuild_render::macro_calls::models::MacroCallEvent; + +/// A typed resource reference by its `SqlResourceRefKind` value and name. +pub(crate) type ResourceRef = (String, String); + +/// `MacroExpansionFacts`: dependencies, usages, relations, call-site references and the event log. +#[derive(Default)] +pub(crate) struct ExpansionFacts { + pub(crate) dependencies: Vec>, + pub(crate) usages: Vec>, + /// The string's relation table: each typed reference a macro rendered, by placeholder index. + pub(crate) relations: Vec, + pub(crate) argument_references: Vec, + call_site_refs: Vec>, + rendering_approved: Vec>, + event_log: Option>, + memo_blocked: bool, +} + +impl ExpansionFacts { + /// Start recording the consumer-independent facts of one top-level macro call. + pub(crate) fn begin_event_log(&mut self) { + self.event_log = Some(Vec::new()); + self.memo_blocked = false; + } + + /// Stop recording; the call's events and whether it may be replayed. + pub(crate) fn end_event_log(&mut self) -> (Vec, bool) { + ( + self.event_log.take().unwrap_or_default(), + !self.memo_blocked, + ) + } + + /// Record one fact of the executing call, when recording. + pub(crate) fn note_event(&mut self, event: MacroCallEvent) { + if let Some(log) = self.event_log.as_mut() { + log.push(event); + } + } + + /// Mark the executing call as dependent on its consumer, so it is never replayed. + pub(crate) fn block_memo(&mut self) { + self.memo_blocked = true; + } + + /// Record references written as arguments of every macro call being expanded. + pub(crate) fn record_call_site_refs(&mut self, refs: &[ResourceRef]) { + for (kind, name) in refs { + self.note_event(MacroCallEvent::ArgumentReference { + kind: kind.clone(), + name: name.clone(), + }); + } + for reference in refs { + if !self.argument_references.contains(reference) { + self.argument_references.push(reference.clone()); + } + } + for approved in &mut self.call_site_refs { + approved.extend(refs.iter().cloned()); + } + } + + /// Start collecting the references written as one macro call's arguments. + pub(crate) fn open_call_site(&mut self) { + self.call_site_refs.push(HashSet::new()); + } + + /// Stop collecting for the innermost call and return its argument references. + pub(crate) fn close_call_site(&mut self) -> HashSet { + self.call_site_refs.pop().unwrap_or_default() + } + + /// Approve the call-site argument references of the macro about to run. + pub(crate) fn begin_rendering(&mut self, approved: HashSet) { + self.rendering_approved.push(approved); + } + + /// Restore the approvals of the enclosing macro after one macro returns. + pub(crate) fn end_rendering(&mut self) { + let _ = self.rendering_approved.pop(); + } + + /// The placeholder index of an approved reference, or `None` when it renders as its call. + pub(crate) fn relation_index(&mut self, reference: &ResourceRef) -> Option { + if !self + .rendering_approved + .last() + .is_some_and(|approved| approved.contains(reference)) + { + return None; + } + Some(self.relation_slot(reference)) + } + + /// `relations.setdefault(reference, len(relations))`. + pub(crate) fn relation_slot(&mut self, reference: &ResourceRef) -> usize { + if let Some(index) = self.relations.iter().position(|known| known == reference) { + return index; + } + self.relations.push(reference.clone()); + self.relations.len() - 1 + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs new file mode 100644 index 0000000000..19e2f37ea4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs @@ -0,0 +1,256 @@ +//! Diagnostics for reference calls a macro returned: P012 for rejected calls, P006 when enforced. + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyTuple}; +use sqlbuild_analysis::compiled_project::main::text_position::text_position; +use sqlbuild_analysis::compiled_project::models::TextPosition; +use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; +use sqlbuild_sqltext::sql_references::models::{ + InvalidReferenceCall, ReferenceExtraction, ReferenceScan, +}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; + +const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; +const MACRO_GENERATED_REFERENCE_CODE: &str = "P006"; +/// `(SqlReferenceKind value, SqlResourceRefKind value, typed reference function)`. +const EXPLICIT_REFERENCES: [(&str, &str, &str); 3] = [ + ("ref", "model", "__ref"), + ("source", "source", "__source"), + ("seed", "seed", "__seed"), +]; +/// Consumer resource kinds that are compiled resource types of the same value. +const RESOURCE_TYPES: [&str; 3] = ["model", "source", "seed"]; +const MODEL_KIND: &str = "model"; + +/// What the diagnostics are built and reported with. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct GeneratedReferenceClasses<'py> { + compiler_diagnostic: Bound<'py, PyAny>, + report_diagnostic: Bound<'py, PyAny>, + source_location: Bound<'py, PyAny>, + resource_identity: Bound<'py, PyAny>, + generic_syntax: LexicalSyntaxInput, + /// `explicit_references_help(allowed="macro-generated references")`. + generated_help: String, + /// `function(` of every reference kind. + all_markers: Vec, +} + +impl GeneratedReferenceClasses<'_> { + /// `function(` of every reference kind. + pub(crate) fn markers(&self) -> impl Iterator { + self.all_markers.iter().map(String::as_str) + } +} + +/// The expansion a macro's result was returned to: its file, consumer and enforcement. +pub(crate) struct GeneratedOrigin<'a, 'py> { + pub(crate) file_path: &'a Bound<'py, PyAny>, + pub(crate) consumer: &'a Bound<'py, PyAny>, + pub(crate) enforce_explicit: bool, +} + +/// `report_macro_generated_references`. +pub(crate) fn report_generated_references( + classes: &GeneratedReferenceClasses<'_>, + origin: &GeneratedOrigin<'_, '_>, + loaded_macro: &Bound<'_, PyAny>, + result: &str, +) -> PyResult<()> { + if !classes.markers().any(|marker| result.contains(marker)) { + return Ok(()); + } + let syntax: LexicalSyntax = classes.generic_syntax.clone().into(); + let ReferenceExtraction::Extracted(scan) = extract_sql_references(result, &syntax) else { + return Ok(()); + }; + let reporter: Reporter<'_, '_> = Reporter { + classes, + origin, + loaded_macro, + macro_name: loaded_macro.getattr("name")?.extract()?, + resource: resource_kind(classes, origin.consumer)?, + }; + for invalid_call in &scan.invalid_calls { + reporter.rejected_call(invalid_call)?; + } + if origin.enforce_explicit + && EXPLICIT_REFERENCES + .iter() + .any(|(_, _, function)| result.contains(&format!("{function}("))) + { + reporter.generated(&scan)?; + } + Ok(()) +} + +/// The consumer's resource kind when it is a `ResourceIdentity`. +fn resource_kind( + classes: &GeneratedReferenceClasses<'_>, + consumer: &Bound<'_, PyAny>, +) -> PyResult> { + if consumer.is_none() || !consumer.is_instance(&classes.resource_identity)? { + return Ok(None); + } + Ok(Some(( + consumer.getattr("kind")?.getattr("value")?.extract()?, + consumer.getattr("name")?.extract()?, + ))) +} + +struct Reporter<'a, 'py> { + classes: &'a GeneratedReferenceClasses<'py>, + origin: &'a GeneratedOrigin<'a, 'py>, + loaded_macro: &'a Bound<'py, PyAny>, + macro_name: String, + resource: Option<(String, String)>, +} + +impl<'py> Reporter<'_, 'py> { + fn py(&self) -> Python<'py> { + self.loaded_macro.py() + } + + /// `"the model"` for a model consumer, else `"the calling SQL"`. + fn location_label(&self) -> &'static str { + match &self.resource { + Some((kind, _)) if kind == MODEL_KIND => "the model", + _ => "the calling SQL", + } + } + + /// `report_macro_reference_call_syntax` for one rejected call. + fn rejected_call(&self, invalid_call: &InvalidReferenceCall) -> PyResult<()> { + let py: Python<'py> = self.py(); + let name: &str = &self.macro_name; + let corrected: &str = &invalid_call.corrected_call; + let pass_in_help: String = if EXPLICIT_REFERENCES + .iter() + .any(|(kind, _, _)| *kind == invalid_call.kind) + { + format!( + "write {corrected} in {} and pass it in: @{name}({corrected})\n = help: ", + self.location_label() + ) + } else { + String::new() + }; + let fields: Bound<'py, PyDict> = self.fields( + REFERENCE_CALL_SYNTAX_CODE, + &format!("{}, returned by macro {name}()", invalid_call.message), + &format!("{pass_in_help}{}", invalid_call.help), + )?; + fields.set_item("location", self.source_location(&invalid_call.call)?)?; + let owner: String = self.origin.file_path.call_method0("as_posix")?.extract()?; + let key: Bound<'py, PyTuple> = + PyTuple::new(py, [REFERENCE_CALL_SYNTAX_CODE, &owner, &invalid_call.call])?; + self.report(key, &fields) + } + + /// `reject_macro_generated_references`: P006 for each model, source or seed it generated. + fn generated(&self, scan: &ReferenceScan) -> PyResult<()> { + let py: Python<'py> = self.py(); + let consumer_label: String = if self.origin.consumer.is_none() { + format!("'{}'", self.origin.file_path.str()?) + } else { + format!( + "{}:{}", + self.origin + .consumer + .getattr("kind")? + .getattr("value")? + .extract::()?, + self.origin.consumer.getattr("name")?.extract::()? + ) + }; + let mut generated: Vec<(&str, &str, &str)> = Vec::new(); + for reference in &scan.references { + let Some((_, kind, function)) = EXPLICIT_REFERENCES + .iter() + .find(|(reference_kind, _, _)| *reference_kind == reference.kind) + else { + continue; + }; + let entry: (&str, &str, &str) = (kind, reference.name.as_str(), function); + if !generated.contains(&entry) { + generated.push(entry); + } + } + let name: &str = &self.macro_name; + for (kind, reference, function) in generated { + let fields: Bound<'py, PyDict> = self.fields( + MACRO_GENERATED_REFERENCE_CODE, + &format!("{consumer_label} depends on {kind}:{reference} through macro {name}()"), + &format!( + "write the reference in {}, or pass it in: @{name}({function}(\"{reference}\"))\n = help: {}", + self.location_label(), + self.classes.generated_help + ), + )?; + let key: Bound<'py, PyTuple> = PyTuple::new( + py, + [ + MACRO_GENERATED_REFERENCE_CODE, + &consumer_label, + name, + kind, + reference, + ], + )?; + self.report(key, &fields)?; + } + Ok(()) + } + + /// A compile-phase error at the macro file, attributed to the consumer resource. + fn fields(&self, code: &str, message: &str, help: &str) -> PyResult> { + let fields: Bound<'py, PyDict> = PyDict::new(self.py()); + fields.set_item("phase", "compile")?; + fields.set_item("severity", "error")?; + fields.set_item("code", code)?; + fields.set_item("message", message)?; + match &self.resource { + Some((kind, name)) if RESOURCE_TYPES.contains(&kind.as_str()) => { + fields.set_item("resource_type", kind)?; + fields.set_item("resource_name", name)?; + } + _ => {} + } + fields.set_item("path", self.loaded_macro.getattr("relative_path")?)?; + fields.set_item("help", help)?; + Ok(fields) + } + + /// `_macro_source_location`: the call's first occurrence in the macro's source, if any. + fn source_location(&self, call: &str) -> PyResult> { + let source: String = self.loaded_macro.getattr("raw_source")?.extract()?; + let Some(byte_start) = source.find(call) else { + return Ok(self.py().None().into_bound(self.py())); + }; + let start: usize = source[..byte_start].chars().count(); + let begin: TextPosition = text_position(&source, start); + let end: TextPosition = text_position(&source, start + call.chars().count()); + let location: Bound<'py, PyDict> = PyDict::new(self.py()); + location.set_item("path", self.loaded_macro.getattr("relative_path")?)?; + location.set_item("line", begin.line)?; + location.set_item("column", begin.column)?; + location.set_item("end_line", end.line)?; + location.set_item("end_column", end.column)?; + self.classes.source_location.call((), Some(&location)) + } + + fn report(&self, key: Bound<'py, PyTuple>, fields: &Bound<'py, PyDict>) -> PyResult<()> { + let arguments: Bound<'py, PyDict> = PyDict::new(self.py()); + arguments.set_item("key", key)?; + arguments.set_item( + "diagnostic", + self.classes.compiler_diagnostic.call((), Some(fields))?, + )?; + let _ = self.classes.report_diagnostic.call((), Some(&arguments))?; + Ok(()) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs index ad5b571b93..8b1d78a6c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/mod.rs @@ -1,5 +1,11 @@ +pub(crate) mod declaration_expansion; pub(crate) mod macro_arguments; +pub(crate) mod macro_call_classes; pub(crate) mod macro_call_memo; pub(crate) mod macro_call_rows; pub(crate) mod macro_calls; +pub(crate) mod macro_evaluation; +pub(crate) mod macro_expansion; +pub(crate) mod macro_expansion_facts; +pub(crate) mod macro_generated_references; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs index d3e3f56667..b57e7ed63e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/registration.rs @@ -2,10 +2,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::render::{macro_arguments, macro_calls}; +use crate::bindings::_helpers::render::{ + declaration_expansion, macro_arguments, macro_calls, macro_expansion, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + declaration_expansion::register(module)?; macro_arguments::register(module)?; macro_calls::register(module)?; + macro_expansion::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs index d421144bd3..cb5c126936 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/evaluation.rs @@ -1,58 +1,15 @@ //! Register the versioned Python boundary for the native Rules engine. use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBytes, PyBytesMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::configuration::main::load; use sqlbuild_rules::constants::API_VERSION; -use sqlbuild_rules::engine::main::{evaluate, evaluate_parsed, parse_parts}; use sqlbuild_rules::models::CatalogueResponse; use sqlbuild_rules::rules::main::{catalogue, selected_codes}; -use std::sync::Mutex; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; -use crate::bindings::models::ParsedRulesRequest; use crate::bindings::types::CompilerDetach; -#[pyfunction] -fn evaluate_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| evaluate::evaluate_json(request_json)) - .map_err(value_error) -} - -#[pyfunction] -fn parse_rules_parts( - py: Python<'_>, - request_json: Bound<'_, PyBytes>, - model_jsons: Vec>, - model_digests: Vec, -) -> PyResult { - let request: &[u8] = request_json.as_bytes(); - let models: Vec<&[u8]> = model_jsons.iter().map(|model| model.as_bytes()).collect(); - let parsed = py - .compiler_detach(|| parse_parts::parse_parts(request, &models, &model_digests)) - .map_err(value_error)?; - Ok(ParsedRulesRequest { - request: Mutex::new(Some(parsed)), - }) -} - -#[pyfunction] -fn evaluate_parsed_rules( - py: Python<'_>, - parsed: &Bound<'_, ParsedRulesRequest>, -) -> PyResult { - let request = parsed - .borrow() - .request - .lock() - .map_err(|_| value_error("rules request lock is poisoned"))? - .take() - .ok_or_else(|| value_error("rules request was already evaluated"))?; - py.compiler_detach(|| evaluate_parsed::evaluate_parsed(request)) - .map_err(value_error) -} - #[pyfunction] fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { py.compiler_detach(|| { @@ -61,14 +18,6 @@ fn run_custom_host_json(py: Python<'_>, spec_json: &str) -> PyResult { .map_err(value_error) } -#[pyfunction] -fn finalize_rule_findings_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_rules::engine::main::finalize::finalize_findings_json(request_json) - }) - .map_err(value_error) -} - #[pyfunction] fn load_config_json(project_dir: &str) -> PyResult { compiler_guard(|| { @@ -93,13 +42,9 @@ fn selected_codes_json(request_json: &str) -> PyResult { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(evaluate_json, module)?)?; - module.add_function(wrap_pyfunction!(parse_rules_parts, module)?)?; - module.add_function(wrap_pyfunction!(evaluate_parsed_rules, module)?)?; module.add_function(wrap_pyfunction!(run_custom_host_json, module)?)?; - module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?; module.add_function(wrap_pyfunction!(load_config_json, module)?)?; module.add_function(wrap_pyfunction!(catalogue_json, module)?)?; module.add_function(wrap_pyfunction!(selected_codes_json, module)?)?; - Ok(()) + crate::bindings::_helpers::rules::rows::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index 5b3f06f034..6fdc2696b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::rules::relation_keys; use crate::bindings::types::CompilerDetach; #[pyfunction] @@ -100,5 +101,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_batch_json, module)?)?; - Ok(()) + relation_keys::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 0d2a1be28f..c10cf564a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod evaluation; pub(crate) mod lint; +pub(crate) mod relation_keys; +pub(crate) mod rows; pub(crate) mod skills; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs new file mode 100644 index 0000000000..4e61994603 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -0,0 +1,181 @@ +//! Declared unique keys of project relations, derived natively from their key facts. + +use std::collections::BTreeMap; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_rules::sql_quality::main::declared_relation_keys::declared_relation_keys as derive_keys; +use sqlbuild_rules::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; + +const MATERIALIZED_CONFIG: &str = "materialized"; +const INCREMENTAL_MODE_CONFIG: &str = "incremental_mode"; +const INCREMENTAL_STRATEGY_CONFIG: &str = "incremental_strategy"; +const CURSOR_CONFIG: &str = "cursor"; +const UNIQUE_KEY_CONFIG: &str = "unique_key"; +const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; + +/// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. +pub(crate) type RelationKeyRow = ( + String, + String, + String, + String, + String, + bool, + Option>, + Vec<(String, Vec, bool)>, + Vec<(String, Option)>, +); + +/// The key row of one relation from its config values and declared column and audit specs. +pub(crate) fn relation_key_row( + kind: &str, + name: String, + values: &Bound<'_, PyDict>, + declared: Option<(&Bound<'_, PyAny>, &Bound<'_, PyAny>)>, +) -> PyResult { + let mut columns: Vec<(String, Vec, bool)> = Vec::new(); + let mut audits: Vec<(String, Option)> = Vec::new(); + if let Some((declared_columns, declared_audits)) = declared { + for column in declared_columns.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let nullable: Option> = column.getattr_opt("nullable")?; + columns.push(( + column.getattr("name")?.str()?.extract()?, + audit_names(column.getattr_opt("audits")?)?, + nullable.is_some_and(|value| value.is(PyBool::new(column.py(), false))), + )); + } + for audit in declared_audits.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let argument: Option> = audit + .getattr("arguments")? + .downcast::()? + .get_item(UNIQUE_AUDIT_COLUMN_ARGUMENT)?; + audits.push(( + audit.getattr("definition_name")?.extract()?, + argument + .filter(|value| value.is_instance_of::()) + .map(|value| value.extract()) + .transpose()?, + )); + } + } + Ok(( + kind.to_owned(), + name, + config_text(values, MATERIALIZED_CONFIG)?, + config_text(values, INCREMENTAL_MODE_CONFIG)?, + config_text(values, INCREMENTAL_STRATEGY_CONFIG)?, + values + .get_item(CURSOR_CONFIG)? + .is_some_and(|value| value.is_truthy().unwrap_or(false)), + unique_key(values)?, + columns, + audits, + )) +} + +fn audit_names(audits: Option>) -> PyResult> { + let Some(audits) = audits.filter(|audits| !audits.is_none()) else { + return Ok(Vec::new()); + }; + let mut names: Vec = Vec::new(); + for audit in audits.try_iter()? { + names.push(audit?.getattr("definition_name")?.extract()?); + } + Ok(names) +} + +/// `str(values.get(key, ""))`. +fn config_text(values: &Bound<'_, PyDict>, key: &str) -> PyResult { + match values.get_item(key)? { + Some(value) => value.str()?.extract(), + None => Ok(String::new()), + } +} + +/// `unique_key` as a name list, when it is a name or a non-empty list or tuple of names. +fn unique_key(values: &Bound<'_, PyDict>) -> PyResult>> { + let Some(value) = values.get_item(UNIQUE_KEY_CONFIG)? else { + return Ok(None); + }; + if value.is_instance_of::() { + return Ok(Some(vec![value.extract()?])); + } + if !(value.is_instance_of::() || value.is_instance_of::()) { + return Ok(None); + } + let mut names: Vec = Vec::new(); + for item in value.try_iter()? { + let item: Bound<'_, PyAny> = item?; + if !item.is_instance_of::() { + return Ok(None); + } + names.push(item.str()?.extract()?); + } + Ok(Some(names)) +} + +/// The key row of a seed, source or hand-built model from its config values and schema entry. +#[pyfunction(name = "relation_key_row")] +#[pyo3(signature = (kind, name, values, entry))] +fn entry_relation_key_row( + kind: &str, + name: String, + values: &Bound<'_, PyDict>, + entry: &Bound<'_, PyAny>, +) -> PyResult { + if entry.is_none() { + return relation_key_row(kind, name, values, None); + } + let columns: Bound<'_, PyAny> = declared_specs(entry, "columns")?; + let audits: Bound<'_, PyAny> = declared_specs(entry, "audits")?; + relation_key_row(kind, name, values, Some((&columns, &audits))) +} + +/// `getattr(entry, name, None) or ()`. +fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult> { + match entry.getattr_opt(name)? { + Some(specs) if specs.is_truthy()? => Ok(specs), + _ => Ok(PyTuple::empty(entry.py()).into_any()), + } +} + +/// Keys that prove one row per key, by `kind:name` identity, from relation key rows. +#[pyfunction] +fn declared_relation_keys( + py: Python<'_>, + rows: Vec, +) -> BTreeMap>> { + let relations: Vec = rows.into_iter().map(declared_relation).collect(); + py.detach(|| derive_keys(&relations)) +} + +fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { + let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits) = row; + DeclaredRelation { + kind, + name, + materialized, + incremental_mode: mode, + incremental_strategy: strategy, + has_cursor, + unique_key, + columns: columns + .into_iter() + .map(|(name, audits, not_nullable)| DeclaredKeyColumn { + name, + audits, + not_nullable, + }) + .collect(), + audits, + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(entry_relation_key_row, module)?)?; + module.add_function(wrap_pyfunction!(declared_relation_keys, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs new file mode 100644 index 0000000000..304cedf893 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -0,0 +1,447 @@ +//! Build and evaluate the built-in rules request natively from compiled-project rows. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Mutex; + +use pyo3::exceptions::{PyOSError, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyBytes, PyBytesMethods, PyDict, PyDictMethods}; +use pyo3::{FromPyObject, pyclass, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::engine::main::build_models::build_models; +use sqlbuild_rules::engine::main::evaluate_rows::evaluate_rows; +use sqlbuild_rules::engine::main::finalize_rows::finalize_rows; +use sqlbuild_rules::engine::main::project_model_rows::project_model_rows; +use sqlbuild_rules::errors::RowsError; +use sqlbuild_rules::models::{ + Declaration, DeclarationMember, DirectTestResourceKind, EvaluateRequest, Fault, Model, + ModelRow, ModelRowsContext, RowsEvaluation, RowsRequest, RulesConfig, SqlScenarioFact, + SqlTestCteFact, SqlTestFact, SqlTestMode, SqlTestParameterFact, SqlTestParameterValueFact, + TestedResource, +}; + +use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::values::plain_value; +use crate::bindings::types::CompilerDetach; + +const LEFT_ARGUMENT: &str = "left"; +const RIGHT_ARGUMENT: &str = "right"; + +type FaultRow = (bool, String, String, u64, u64, String, String); +type CteRow = (String, String); +type ModelTestPayloadPy = (Vec, Vec, Vec, bool, bool); +type TestNamesRow = (Vec, Vec, Vec, Vec); +type BuiltCounts = (usize, usize, usize); +type EvaluationRow = (Vec, Vec, u64, u64, u64, u64, bool); + +/// Name, relative path, members, value, value type and rendering of one declaration. +#[derive(FromPyObject)] +struct DeclarationRowPy<'py>( + String, + String, + Vec<(String, Bound<'py, PyAny>)>, + Bound<'py, PyAny>, + Option, + Option, +); + +#[derive(FromPyObject)] +struct SqlTestRowPy<'py>( + String, + String, + u64, + String, + Option, + String, + TestNamesRow, + Vec<(String, String)>, + Option, + Option>, +); + +#[derive(FromPyObject)] +struct SqlTestCaseRowPy<'py>( + Option, + String, + Option, + Option, + Vec<(String, String, bool)>, + Vec<(String, Bound<'py, PyAny>)>, +); + +#[derive(FromPyObject)] +struct ScenarioRowPy( + String, + String, + String, + Option, + Vec, + Vec, + Vec, + Vec, +); + +/// The request's project facts, which models, tests and scenarios are built beside. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct ProjectRowsPy<'py> { + header_json: Bound<'py, PyBytes>, + memo: Option<(String, String)>, + sql_analysis_enabled: bool, + include_type_proof: bool, + attached_audit_targets: Vec, + public_enums: Vec>, + public_constants: Vec>, + initial_findings: Vec, + types_equal: Bound<'py, PyAny>, +} + +/// A natively built rules request held until it is evaluated once. +#[pyclass(module = "sqlbuild._native")] +struct NativeRulesRequest { + request: Mutex>, +} + +#[pyfunction] +fn build_rules_request<'py>( + mut project: ProjectRowsPy<'py>, + models: (PyRef<'py, NativeCompiledProject>, Vec), + sql_tests: Vec>, + scenarios: Vec, +) -> PyResult<(NativeRulesRequest, BuiltCounts)> { + compiler_guard(|| { + let mut request: EvaluateRequest = + serde_json::from_slice(project.header_json.as_bytes()).map_err(value_error)?; + request.sql_tests = sql_tests + .into_iter() + .map(sql_test) + .collect::>()?; + request.sql_scenarios = scenarios.into_iter().map(scenario).collect(); + request.public_enums = declarations(std::mem::take(&mut project.public_enums))?; + request.public_constants = declarations(std::mem::take(&mut project.public_constants))?; + request.initial_findings = std::mem::take(&mut project.initial_findings) + .into_iter() + .map(fault) + .collect(); + let rows: Vec = + project_model_rows(models.0.facts(), &models.1).map_err(value_error)?; + request.models = built_models(&project, &request, rows)?; + let counts: BuiltCounts = ( + request.models.len(), + request.sql_tests.len(), + request.sql_scenarios.len(), + ); + let memo: Option<(String, PathBuf)> = project + .memo + .take() + .map(|(identity, path)| (identity, PathBuf::from(path))); + Ok(( + NativeRulesRequest { + request: Mutex::new(Some(RowsRequest { request, memo })), + }, + counts, + )) + }) +} + +#[pyfunction] +fn evaluate_rules_request( + py: Python<'_>, + request: &Bound<'_, NativeRulesRequest>, +) -> PyResult { + let request: RowsRequest = request + .borrow() + .request + .lock() + .map_err(|_| value_error("rules request lock is poisoned"))? + .take() + .ok_or_else(|| value_error("rules request was already evaluated"))?; + let evaluation: RowsEvaluation = py + .compiler_detach(|| Ok(evaluate_rows(request))) + .map_err(value_error)? + .map_err(|error| match error { + RowsError::Rules(message) => value_error(message), + RowsError::Io(error) => PyOSError::new_err(error.to_string()), + })?; + Ok(( + evaluation.faults.into_iter().map(fault_row).collect(), + evaluation.selected_codes, + evaluation.evaluated_models, + evaluation.cache_hits, + evaluation.cache_misses, + evaluation.built_in_ms, + evaluation.reused, + )) +} + +/// The findings one finalisation applies the exception policy to. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct FinalizeRowsPy<'py> { + project_dir: String, + config_json: Bound<'py, PyBytes>, + evaluated_codes: Vec, + findings: Vec, +} + +#[pyfunction] +fn finalize_rule_findings_rows( + py: Python<'_>, + request: FinalizeRowsPy<'_>, +) -> PyResult> { + let config: RulesConfig = serde_json::from_slice(request.config_json.as_bytes()) + .map_err(|error| value_error(format!("invalid findings request: {error}")))?; + let FinalizeRowsPy { + project_dir, + evaluated_codes, + findings, + .. + } = request; + let faults: Vec = findings.into_iter().map(fault).collect(); + let finalized: Vec = py + .compiler_detach(|| finalize_rows(project_dir, config, &evaluated_codes, faults)) + .map_err(value_error)?; + Ok(finalized.into_iter().map(fault_row).collect()) +} + +/// `left=` and `right=`: the callback is `types_equal`, whose arguments are keyword-only. +fn type_arguments<'py>(py: Python<'py>, left: &str, right: &str) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item(LEFT_ARGUMENT, left)?; + arguments.set_item(RIGHT_ARGUMENT, right)?; + Ok(arguments) +} + +fn built_models( + project: &ProjectRowsPy<'_>, + request: &EvaluateRequest, + rows: Vec, +) -> PyResult> { + let mut python_error: Option = None; + let fallback = |left: &str, right: &str| -> Result { + match type_arguments(project.types_equal.py(), left, right) + .and_then(|arguments| project.types_equal.call((), Some(&arguments))) + .and_then(|equal| equal.extract::()) + { + Ok(equal) => Ok(equal), + Err(error) => { + let message: String = error.to_string(); + python_error = Some(error); + Err(message) + } + } + }; + let built: Result, String> = build_models( + rows, + ModelRowsContext { + sql_analysis_enabled: project.sql_analysis_enabled, + include_type_proof: project.include_type_proof, + dialect: &request.dialect, + attached_audit_targets: &project.attached_audit_targets, + sql_tests: &request.sql_tests, + }, + fallback, + ); + built.map_err(|message| python_error.take().unwrap_or_else(|| value_error(message))) +} + +fn declarations(rows: Vec>) -> PyResult> { + rows.into_iter().map(declaration).collect() +} + +fn declaration(row: DeclarationRowPy<'_>) -> PyResult { + let DeclarationRowPy(name, relative_path, members, value, value_type, render_as) = row; + Ok(Declaration { + name, + relative_path, + members: members + .into_iter() + .map(declaration_member) + .collect::>()?, + value: Some(plain_value(&value)?).filter(|value| !value.is_null()), + value_type, + render_as, + }) +} + +fn declaration_member((name, value): (String, Bound<'_, PyAny>)) -> PyResult { + Ok(DeclarationMember { + name, + value: plain_value(&value)?, + }) +} + +fn sql_test(row: SqlTestRowPy<'_>) -> PyResult { + let SqlTestRowPy( + source_path, + ownership_root, + block_index, + name, + explicit_name, + mode, + (expected_model_names, assertion_names, assertion_target_model_names, target_model_names), + tested_resources, + model_payload, + case, + ) = row; + let (authored_ctes, expected_ctes, assertion_ctes, has_macro_mocks, has_model_query_overrides) = + model_payload.unwrap_or_default(); + let mut fact = SqlTestFact { + source_path, + ownership_root, + block_index, + name, + explicit_name, + parent_name: None, + case_name: None, + case_index: None, + case_fingerprint: None, + parameter_schema: Vec::new(), + parameters: Vec::new(), + mode: test_mode(&mode)?, + expected_model_names, + assertion_names, + assertion_target_model_names, + target_model_names, + tested_resources: tested_resources + .into_iter() + .map(|(kind, name)| { + Ok(TestedResource { + kind: tested_kind(&kind)?, + name, + }) + }) + .collect::>()?, + authored_ctes: ctes(authored_ctes), + expected_ctes: ctes(expected_ctes), + assertion_ctes: ctes(assertion_ctes), + has_macro_mocks, + has_model_query_overrides, + empty_input_only: false, + }; + if let Some(SqlTestCaseRowPy( + parent_name, + case_name, + case_index, + case_fingerprint, + schema, + values, + )) = case + { + let types: HashMap<&str, &str> = schema + .iter() + .map(|(name, value_type, _)| (name.as_str(), value_type.as_str())) + .collect(); + fact.parameters = values + .into_iter() + .map(|(name, value)| { + let value_type: String = types + .get(name.as_str()) + .ok_or_else(|| PyValueError::new_err(format!("unknown test parameter {name}")))? + .to_string(); + Ok(SqlTestParameterValueFact { + name, + value_type, + value: plain_value(&value)?, + }) + }) + .collect::>()?; + fact.parameter_schema = schema + .into_iter() + .map(|(name, value_type, nullable)| SqlTestParameterFact { + name, + value_type, + nullable, + }) + .collect(); + fact.parent_name = parent_name; + fact.case_name = Some(case_name); + fact.case_index = case_index; + fact.case_fingerprint = case_fingerprint; + } + Ok(fact) +} + +fn ctes(rows: Vec) -> Vec { + rows.into_iter() + .map(|(name, sql)| SqlTestCteFact { name, sql }) + .collect() +} + +fn scenario(row: ScenarioRowPy) -> SqlScenarioFact { + let ScenarioRowPy( + source_path, + ownership_root, + name, + description, + expected_model_names, + assertion_names, + assertion_target_model_names, + target_model_names, + ) = row; + SqlScenarioFact { + source_path, + ownership_root, + name, + description, + expected_model_names, + assertion_names, + assertion_target_model_names, + target_model_names, + } +} + +fn test_mode(value: &str) -> PyResult { + match value { + "model" => Ok(SqlTestMode::Model), + "macro" => Ok(SqlTestMode::Macro), + "udf" => Ok(SqlTestMode::Udf), + "table_fn" => Ok(SqlTestMode::TableFn), + _ => Err(value_error(format!("unknown SQL test mode {value}"))), + } +} + +fn tested_kind(value: &str) -> PyResult { + match value { + "macro" => Ok(DirectTestResourceKind::Macro), + "udf" => Ok(DirectTestResourceKind::Udf), + "table_fn" => Ok(DirectTestResourceKind::TableFn), + _ => Err(value_error(format!("unknown tested resource kind {value}"))), + } +} + +fn fault((unevaluated, code, path, line, column, message, remediation): FaultRow) -> Fault { + Fault { + unevaluated, + code, + path, + line, + column, + message, + remediation, + } +} + +fn fault_row(fault: Fault) -> FaultRow { + ( + fault.unevaluated, + fault.code, + fault.path, + fault.line, + fault.column, + fault.message, + fault.remediation, + ) +} + +/// A plain Python value as the rules request's JSON value, as orjson with `default=str` encodes it. +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_rules_request, module)?)?; + module.add_function(wrap_pyfunction!(evaluate_rules_request, module)?)?; + module.add_function(wrap_pyfunction!(finalize_rule_findings_rows, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs new file mode 100644 index 0000000000..2516a544d7 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -0,0 +1,164 @@ +//! Compile usage facts: schema parent and enum usages with S006, and the usage merge. + +use std::collections::HashMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_scopes::compile_usages::main::distinct_positions::distinct_positions; +use sqlbuild_scopes::compile_usages::main::merged_dependencies::merged_dependencies; +use sqlbuild_scopes::compile_usages::main::schema_scope_usages::schema_scope_usages; +use sqlbuild_scopes::compile_usages::models::{ + DependencySource, InaccessibleDiagnostic, NamedRecord, SchemaDeclaration, SchemaUsages, +}; +use sqlbuild_scopes::scope_index::models::{ + DeclarationKind, DeclaredScope, ResourceIdentity, ResourceKind, ScopeKind, +}; + +use crate::bindings::_helpers::compiled_project::model_loop_named::scope_path_message; + +type IdentityRow = (String, String); +/// A declaration's kind, name and private owner; a resource's kind and name with no owner. +type QualifiedRow = (String, String, Option); +/// Consumer, declaration, usage kind, what it is used through, and the enum member. +type UsageRow = ( + QualifiedRow, + QualifiedRow, + String, + Option, + Option, +); +/// Path, scope, private owner, owning path and ownership root path of a named record. +type RecordRow = (String, String, Option, Option, String); +/// Name, the schema it extends, and each column's name and type. +type SchemaRow = (String, Option, Vec<(String, Option)>); +/// Report key, message, path and help of one S006. +type DiagnosticRow = (Vec, String, String, String); +/// Usages as `(consumer, declaration)` record positions, S006s, else a scope path error. +type SchemaUsageRows = (Vec<(usize, usize)>, Vec, Option); +/// `(own, position)`: an own dependency's position, else a distinct usage's. +type SourceRow = (bool, usize); +/// Distinct collected positions, merged sources per declaration, and final usage positions. +type MergeRows = (Vec, Vec>>, Vec); + +fn invalid(what: &str) -> pyo3::PyErr { + PyValueError::new_err(format!("invalid compile usage {what}")) +} + +fn named_record(handle: usize, row: RecordRow) -> PyResult { + let (path, scope, owner, owning_path, ownership_root) = row; + let owner: Option = match owner { + None => None, + Some((kind, name)) => Some(ResourceIdentity { + kind: ResourceKind::parse(&kind).ok_or_else(|| invalid("owner kind"))?, + name, + }), + }; + Ok(NamedRecord { + handle, + path, + declared: DeclaredScope { + scope: ScopeKind::parse(&scope).ok_or_else(|| invalid("scope"))?, + owner, + owning_path, + }, + ownership_root, + }) +} + +fn diagnostic_row(diagnostic: InaccessibleDiagnostic) -> DiagnosticRow { + ( + diagnostic.key, + diagnostic.message, + diagnostic.path, + diagnostic.help, + ) +} + +/// Each schema's parent and enum usages over `records` by `(kind, name)` handle, with S006s. +#[pyfunction] +fn scope_schema_usages( + schemas: Vec, + records: Vec<(IdentityRow, RecordRow)>, +) -> PyResult { + let mut schema_records: HashMap = HashMap::new(); + let mut enum_records: HashMap = HashMap::new(); + for (handle, ((kind, name), row)) in records.into_iter().enumerate() { + let target: &mut HashMap = + match DeclarationKind::parse(&kind).ok_or_else(|| invalid("declaration kind"))? { + DeclarationKind::Schema => &mut schema_records, + DeclarationKind::Enum => &mut enum_records, + _ => continue, + }; + target.insert(name, named_record(handle, row)?); + } + let mut declarations: Vec = Vec::with_capacity(schemas.len()); + for (name, extends, columns) in schemas { + declarations.push(SchemaDeclaration { + name, + extends, + columns, + }); + } + Ok( + match schema_scope_usages(&declarations, &schema_records, &enum_records) { + Ok(SchemaUsages { + usages, + diagnostics, + }) => ( + usages, + diagnostics.into_iter().map(diagnostic_row).collect(), + None, + ), + Err(rejected) => ( + Vec::new(), + Vec::new(), + Some(scope_path_message(&rejected.declared, &rejected.path)), + ), + }, + ) +} + +fn source_row(source: DependencySource) -> SourceRow { + match source { + DependencySource::Own(position) => (true, position), + DependencySource::Usage(position) => (false, position), + } +} + +fn source_rows(sources: Option>) -> Option> { + sources.map(|sources| sources.into_iter().map(source_row).collect()) +} + +/// Distinct collected usages, merged macro dependencies, and the distinct prior-then-new usages. +#[pyfunction] +fn scope_merged_usages( + declarations: Vec<(QualifiedRow, Option>)>, + prior: Vec, + collected: Vec, +) -> MergeRows { + let distinct: Vec = distinct_positions(&collected); + let mut consumed: Vec<(Option, QualifiedRow)> = + Vec::with_capacity(distinct.len()); + for position in &distinct { + let (consumer, declaration, _, _, _) = &collected[*position]; + let by_declaration: Option = DeclarationKind::parse(&consumer.0) + .is_some() + .then(|| consumer.clone()); + consumed.push((by_declaration, declaration.clone())); + } + let merged: Vec>> = merged_dependencies(&declarations, &consumed) + .into_iter() + .map(source_rows) + .collect(); + let mut combined: Vec<&UsageRow> = prior.iter().collect(); + combined.extend(distinct.iter().map(|position| &collected[*position])); + let combined_positions: Vec = distinct_positions(&combined); + (distinct, merged, combined_positions) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scope_schema_usages, module)?)?; + module.add_function(wrap_pyfunction!(scope_merged_usages, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs index 1c646cb59b..c8fe07474e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs @@ -7,6 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{FromPyObject, intern, pyclass, pymethods}; use crate::bindings::_helpers::boundary::panics::{compiler_error, compiler_guard}; +use crate::bindings::_helpers::scopes::declaration_resolution::ResolutionInputs; use sqlbuild_scopes::scope_index::main::classify_resource::classify_resource; use sqlbuild_scopes::scope_index::models::{ ConsumerResource, GrantEntry, VisibilityReason, VisibilityTable, VisibleEntry, @@ -30,20 +31,21 @@ struct PositionRow(Py, Py, Py, u8, Py, u8, u32); /// The Python classes and enum members the contexts are built from. #[derive(FromPyObject)] #[pyo3(from_item_all)] -struct ContextClasses { +pub(crate) struct ContextClasses { visibility_record: Py, - resolution_context: Py, + pub(crate) resolution_context: Py, allocate: Py, reasons: Py, } /// Positions, visibility table and classes of one scope lookup. #[pyclass(frozen, module = "sqlbuild._native")] -struct NativeDeclarationContexts { +pub(crate) struct NativeDeclarationContexts { positions: Vec, table: VisibilityTable, - classes: ContextClasses, + pub(crate) classes: ContextClasses, lookup: LookupMappings, + pub(crate) resolution: ResolutionInputs, } /// The Python lookup mappings a matching resource's private positions and grants come from. @@ -72,8 +74,9 @@ impl NativeDeclarationContexts { positions: Vec, table: TableRow, classes: ContextClasses, - lookup: LookupMappings, + lookups: (LookupMappings, ResolutionInputs), ) -> Self { + let (lookup, resolution) = lookups; let (global, local, inherited) = table; let identity_keys: Vec = positions.iter().map(|row| row.6).collect(); Self { @@ -86,22 +89,27 @@ impl NativeDeclarationContexts { positions, classes, lookup, + resolution, } } - /// Return the declaration context of these matching resources. - fn context<'py>( + /// `resolve_declaration_context`: the context and whether it was projected, not cached. + #[pyo3(signature = (resolver, file_path, resource=None))] + fn resolve<'py>( &self, - py: Python<'py>, - matches: Vec>, - consumer: Bound<'py, PyAny>, - ) -> PyResult> { - compiler_guard(|| self.classified_context(py, matches, consumer)) + resolver: Bound<'py, PyAny>, + file_path: Bound<'py, PyAny>, + resource: Option>, + ) -> PyResult<(Bound<'py, PyAny>, bool)> { + compiler_guard(|| { + let path: String = file_path.str()?.to_string(); + self.resolved_context(&resolver, &path, resource.as_ref()) + }) } } impl NativeDeclarationContexts { - fn classified_context<'py>( + pub(crate) fn classified_context<'py>( &self, py: Python<'py>, matches: Vec>, @@ -270,19 +278,37 @@ impl NativeDeclarationContexts { entry: &VisibleEntry, through: &Option>, ) -> PyResult> { + let none: Bound<'py, PyAny> = py.None().into_bound(py); + let reason: Bound<'py, PyAny> = self + .classes + .reasons + .bind(py) + .get_item(entry.reason.as_str())? + .unwrap_or_else(|| none.clone()); + self.visibility_record_for( + resource, + self.positions[entry.position].0.bind(py), + &reason, + through.as_ref().unwrap_or(&none), + ) + } + + /// A `VisibilityRecord` built without running its dataclass initializer. + pub(crate) fn visibility_record_for<'py>( + &self, + resource: &Bound<'py, PyAny>, + declaration: &Bound<'py, PyAny>, + reason: &Bound<'py, PyAny>, + through: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = resource.py(); let class: &Bound<'py, PyAny> = self.classes.visibility_record.bind(py); let record: Bound<'py, PyAny> = self.classes.allocate.bind(py).call1((class,))?; let attributes: Bound<'py, PyDict> = record.getattr("__dict__")?.downcast_into::()?; attributes.set_item("resource", resource)?; - attributes.set_item("declaration", self.positions[entry.position].0.bind(py))?; - attributes.set_item( - "reason", - self.classes - .reasons - .bind(py) - .get_item(entry.reason.as_str())?, - )?; + attributes.set_item("declaration", declaration)?; + attributes.set_item("reason", reason)?; attributes.set_item("through", through)?; Ok(record) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs new file mode 100644 index 0000000000..96947690b0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs @@ -0,0 +1,373 @@ +//! `resolve_declaration_context`: a path's or resource's declaration context, cached per folder. + +use std::path::Path; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{FromPyObject, intern}; +use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::parse_identity::parse_identity; +use sqlbuild_scopes::scope_index::models::{ + DeclarationIdentity, IdentityFailure, QualifiedIdentity, ResourceIdentity, ScopePathFailure, +}; + +use crate::bindings::_helpers::scopes::declaration_contexts::NativeDeclarationContexts; + +const PATH_CACHE_PREFIX: &str = ""; +const IDENTITY_SEPARATOR: char = ':'; +const POSIX_SEPARATOR: char = '/'; +const CURRENT_FOLDER: &str = "."; +const DECLARATION_STEM: &str = "__declaration__.sql"; +const PATH_RESOURCE_PREFIX: &str = ", + resources: Py, + declarations: Py, + index_declarations: Py, + resource_record: Py, + ownership_root: Py, + resource_identity: Py, + declaration_identity: Py, + resource_kind: Py, + declaration_kind: Py, + private_scope: Py, + identity_error: Py, + path_error: Py, +} + +/// The resources a context is classified for and the consumer it is recorded for. +type Matches<'py> = (Vec>, Bound<'py, PyAny>); + +impl NativeDeclarationContexts { + /// The context of `file_path` or `resource`, and whether it was projected rather than cached. + pub(crate) fn resolved_context<'py>( + &self, + resolver: &Bound<'py, PyAny>, + file_path: &str, + resource: Option<&Bound<'py, PyAny>>, + ) -> PyResult<(Bound<'py, PyAny>, bool)> { + let py: Python<'py> = resolver.py(); + let inputs: &ResolutionInputs = &self.resolution; + let target: String = target_path(resolver, file_path)?; + let empty: Bound<'py, PyTuple> = PyTuple::empty(py); + let resources: Vec> = match resource { + Some(resource) => inputs + .resources + .bind(py) + .call_method1(intern!(py, "get"), (resource, &empty))?, + None => inputs + .resources_by_path + .bind(py) + .call_method1(intern!(py, "get"), (&target, &empty))?, + } + .try_iter()? + .collect::>()?; + let cache: Bound<'py, PyDict> = resolver + .getattr(intern!(py, "contexts_by_directory"))? + .downcast_into()?; + let specific: Bound<'py, PyAny> = resolver.getattr(intern!(py, "resource_specific"))?; + let mut cache_key: Option> = None; + if let [record] = resources.as_slice() + && !specific.is_none() + && !specific.contains(record.getattr(intern!(py, "identity"))?)? + { + let path: String = record.getattr(intern!(py, "path"))?.extract()?; + let root: Bound<'py, PyAny> = record + .getattr(intern!(py, "ownership_root"))? + .getattr(intern!(py, "path"))?; + let key: Bound<'py, PyTuple> = PyTuple::new( + py, + [PyString::new(py, &posix_parent(&path)).into_any(), root], + )?; + if let Some(cached) = cache.get_item(&key)? { + let consumer: Bound<'py, PyAny> = record.getattr(intern!(py, "identity"))?; + return Ok((self.rebound_context(&cached, &consumer)?, false)); + } + cache_key = Some(key); + } else if resources.is_empty() && resource.is_none() { + let key: Bound<'py, PyTuple> = PyTuple::new(py, [PATH_CACHE_PREFIX, target.as_str()])?; + if let Some(cached) = cache.get_item(&key)? { + return Ok((cached, false)); + } + cache_key = Some(key); + } + let parses_identity: bool = resource.is_none() && target.contains(IDENTITY_SEPARATOR); + let (matches, consumer): Matches<'py> = if resources.is_empty() || parses_identity { + self.queried_matches(py, &target, resource)? + } else { + let consumer: Bound<'py, PyAny> = match resource { + Some(resource) => resource.clone(), + None => resources[0].getattr(intern!(py, "identity"))?, + }; + (resources, consumer) + }; + let context: Bound<'py, PyAny> = self.classified_context(py, matches, consumer)?; + if let Some(key) = cache_key { + cache.set_item(key, &context)?; + } + Ok((context, true)) + } + + /// `_queried_matches`: the resources a scope query matches, or the path's lexical record. + fn queried_matches<'py>( + &self, + py: Python<'py>, + target: &str, + resource: Option<&Bound<'py, PyAny>>, + ) -> PyResult> { + let inputs: &ResolutionInputs = &self.resolution; + let empty: Bound<'py, PyTuple> = PyTuple::empty(py); + let lookup = + |mapping: &Py, key: &Bound<'py, PyAny>| -> PyResult>> { + mapping + .bind(py) + .call_method1(intern!(py, "get"), (key, &empty))? + .try_iter()? + .collect() + }; + let (matches, declaration_matches): (Vec>, usize) = match resource { + Some(resource) => (lookup(&inputs.resources, resource)?, 0), + None if target.contains(IDENTITY_SEPARATOR) => match self.identity(py, target)? { + QualifiedIdentity::Resource(identity) => ( + lookup(&inputs.resources, &self.resource_object(py, &identity)?)?, + 0, + ), + QualifiedIdentity::Declaration(identity) => { + let declaration: Bound<'py, PyAny> = self.declaration_object(py, &identity)?; + ( + Vec::new(), + lookup(&inputs.declarations, &declaration)?.len(), + ) + } + }, + None => { + let normalized: String = normalize_scope_path(target) + .map_err(|failure| self.path_error(py, failure, target))?; + let key: Bound<'py, PyAny> = PyString::new(py, &normalized).into_any(); + (lookup(&inputs.resources_by_path, &key)?, 0) + } + }; + if !matches.is_empty() || declaration_matches > 0 { + let consumer: Bound<'py, PyAny> = match (resource, matches.first()) { + (Some(resource), _) => resource.clone(), + (None, Some(first)) => first.getattr(intern!(py, "identity"))?, + (None, None) => py.None().into_bound(py), + }; + return Ok((matches, consumer)); + } + let lexical: String = self.lexical_path(py, target)?; + let identity: Bound<'py, PyAny> = inputs.resource_identity.bind(py).call1(( + inputs.resource_kind.bind(py).call1((MODEL_KIND,))?, + format!("{PATH_RESOURCE_PREFIX}{lexical}{PATH_RESOURCE_SUFFIX}"), + ))?; + let root: Bound<'py, PyAny> = inputs.ownership_root.bind(py).call1((CURRENT_FOLDER,))?; + let record: Bound<'py, PyAny> = inputs + .resource_record + .bind(py) + .call1((identity, lexical, root))?; + Ok((vec![record], py.None().into_bound(py))) + } + + /// The lexical path of the first public declaration defined at `target`, or `target`. + fn lexical_path(&self, py: Python<'_>, target: &str) -> PyResult { + let inputs: &ResolutionInputs = &self.resolution; + let private: &Bound<'_, PyAny> = inputs.private_scope.bind(py); + for record in inputs.index_declarations.bind(py).try_iter()? { + let record: Bound<'_, PyAny> = record?; + let path: String = record.getattr(intern!(py, "path"))?.extract()?; + if path != target || record.getattr(intern!(py, "scope"))?.is(private) { + continue; + } + let owning_path: Option = + record.getattr(intern!(py, "owning_path"))?.extract()?; + let owner: String = owning_path + .filter(|path| !path.is_empty()) + .unwrap_or_else(|| CURRENT_FOLDER.to_owned()); + return Ok(posix_path(&format!( + "{owner}{POSIX_SEPARATOR}{DECLARATION_STEM}" + ))); + } + Ok(target.to_owned()) + } + + fn identity(&self, py: Python<'_>, value: &str) -> PyResult { + parse_identity(value).map_err(|failure| { + let shown = |text: &str| -> String { + PyString::new(py, text) + .repr() + .map_or_else(|_| text.to_owned(), |repr| repr.to_string()) + }; + let message: String = match failure { + IdentityFailure::NoName => { + format!("Qualified identity has no name: {}", shown(value)) + } + IdentityFailure::UnknownKind(kind) => { + format!("Unknown qualified identity kind: {}", shown(&kind)) + } + IdentityFailure::InvalidPrivate => { + format!("Invalid private identity: {}", shown(value)) + } + IdentityFailure::Unqualified => format!( + "Expected a kind-qualified identity such as 'model:name': {}", + shown(value) + ), + }; + raised(self.resolution.identity_error.bind(py), &message) + }) + } + + fn path_error(&self, py: Python<'_>, failure: ScopePathFailure, path: &str) -> PyErr { + let message: String = match failure { + ScopePathFailure::NotProjectRelative => { + format!("Scope path must be project-relative: {path}") + } + ScopePathFailure::EscapesProject => format!("Scope path escapes the project: {path}"), + }; + raised(self.resolution.path_error.bind(py), &message) + } + + fn resource_object<'py>( + &self, + py: Python<'py>, + identity: &ResourceIdentity, + ) -> PyResult> { + let inputs: &ResolutionInputs = &self.resolution; + inputs.resource_identity.bind(py).call1(( + inputs + .resource_kind + .bind(py) + .call1((identity.kind.as_str(),))?, + identity.name.as_str(), + )) + } + + fn declaration_object<'py>( + &self, + py: Python<'py>, + identity: &DeclarationIdentity, + ) -> PyResult> { + let inputs: &ResolutionInputs = &self.resolution; + let owner: Bound<'py, PyAny> = match &identity.owner { + Some(owner) => self.resource_object(py, owner)?, + None => py.None().into_bound(py), + }; + inputs.declaration_identity.bind(py).call1(( + inputs + .declaration_kind + .bind(py) + .call1((identity.kind.as_str(),))?, + identity.name.as_str(), + owner, + )) + } + + /// `_rebind_declaration_context`: fresh mappings, visibility rebound to `consumer`. + pub(crate) fn rebound_context<'py>( + &self, + context: &Bound<'py, PyAny>, + consumer: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = context.py(); + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in COPIED_FIELDS { + let mapping: Bound<'py, PyDict> = context.getattr(field)?.downcast_into()?; + fields.set_item(field, mapping.copy()?)?; + } + for field in VISIBILITY_FIELDS { + let mapping: Bound<'py, PyDict> = context.getattr(field)?.downcast_into()?; + fields.set_item(field, self.rebound_visibility(&mapping, consumer)?)?; + } + fields.set_item(intern!(py, "consumer"), consumer)?; + self.classes + .resolution_context + .bind(py) + .call((), Some(&fields)) + } + + /// `_rebind_visibility`: each record with `consumer` as its resource. + pub(crate) fn rebound_visibility<'py>( + &self, + records_by_name: &Bound<'py, PyDict>, + consumer: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = records_by_name.py(); + let rebound: Bound<'py, PyDict> = PyDict::new(py); + for (name, records) in records_by_name.iter() { + let mut items: Vec> = Vec::new(); + for record in records.try_iter()? { + let record: Bound<'py, PyAny> = record?; + items.push(self.visibility_record_for( + consumer, + &record.getattr(intern!(py, "declaration"))?, + &record.getattr(intern!(py, "reason"))?, + &record.getattr(intern!(py, "through"))?, + )?); + } + rebound.set_item(name, PyTuple::new(py, items)?)?; + } + Ok(rebound) + } +} + +/// Mapping fields `_rebind_declaration_context` copies unchanged. +const COPIED_FIELDS: [&str; 7] = [ + "enums", + "constants", + "macros", + "macro_records", + "inaccessible_enums", + "inaccessible_constants", + "inaccessible_macros", +]; +/// Visibility fields `_rebind_declaration_context` rebinds to the consumer. +const VISIBILITY_FIELDS: [&str; 3] = ["enum_visibility", "constant_visibility", "macro_visibility"]; + +/// `file_path` relative to the resolver's project directory when it is absolute under it. +fn target_path(resolver: &Bound<'_, PyAny>, file_path: &str) -> PyResult { + let project_dir: Bound<'_, PyAny> = resolver.getattr(intern!(resolver.py(), "project_dir"))?; + let path: &Path = Path::new(file_path); + if project_dir.is_none() || !path.is_absolute() { + return Ok(file_path.to_owned()); + } + let project_dir: String = project_dir.str()?.to_string(); + Ok(match path.strip_prefix(&project_dir) { + Ok(relative) if relative.as_os_str().is_empty() => CURRENT_FOLDER.to_owned(), + Ok(relative) => relative.to_string_lossy().into_owned(), + Err(_) => file_path.to_owned(), + }) +} + +/// `Path(path).parent.as_posix()` of a normalized relative path. +fn posix_parent(path: &str) -> String { + match path.rfind(POSIX_SEPARATOR) { + Some(index) => posix_path(&path[..index]), + None => CURRENT_FOLDER.to_owned(), + } +} + +/// `Path(path).as_posix()`: empty and `.` components dropped. +fn posix_path(path: &str) -> String { + let absolute: bool = path.starts_with(POSIX_SEPARATOR); + let parts: Vec<&str> = path + .split(POSIX_SEPARATOR) + .filter(|part| !part.is_empty() && *part != CURRENT_FOLDER) + .collect(); + match (absolute, parts.is_empty()) { + (true, _) => format!("{POSIX_SEPARATOR}{}", parts.join("/")), + (false, true) => CURRENT_FOLDER.to_owned(), + (false, false) => parts.join("/"), + } +} + +fn raised(class: &Bound<'_, PyAny>, message: &str) -> PyErr { + match class.call1((message,)) { + Ok(instance) => PyErr::from_value(instance), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs index 01543fdf62..a42c417031 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -1,4 +1,12 @@ +pub(crate) mod compile_usages; pub(crate) mod declaration_contexts; +pub(crate) mod declaration_resolution; +pub(crate) mod placement; pub(crate) mod registration; pub(crate) mod relationship_names; +pub(crate) mod relationship_scan; +pub(crate) mod scope_cache; pub(crate) mod scope_index; +pub(crate) mod scope_lookup; +pub(crate) mod scope_report_rows; +pub(crate) mod scope_reports; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs new file mode 100644 index 0000000000..54c646f950 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs @@ -0,0 +1,196 @@ +//! Unused-declaration and exact-placement rules over a usage-complete scope index's rows. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_scopes::placement::main::declaration_directory::declaration_directory; +use sqlbuild_scopes::placement::main::placement_diagnostics::placement_diagnostics; +use sqlbuild_scopes::placement::main::relocate_for_resource_move::relocate_for_resource_move; +use sqlbuild_scopes::placement::models::{ + PlacementDiagnostic, PlacementIndex, PlacementUsage, Relocation, +}; +use sqlbuild_scopes::scope_index::models::{ + DeclarationIdentity, DeclarationKind, DeclarationRecord, OwnershipRoot, OwnershipRootKind, + QualifiedIdentity, ResourceIdentity, ResourceKind, ResourceRecord, ScopeKind, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::types::CompilerDetach; + +pub(crate) type IdentityRow = (String, String); +/// A declaration's kind, name and private owner; a resource's kind and name with no owner. +pub(crate) type QualifiedRow = (String, String, Option); +/// An ownership root's path, kind and resource kind. +pub(crate) type RootRow = (String, String, Option); +pub(crate) type ResourceRow = (IdentityRow, String, RootRow); +/// Identity, path, line, column, scope, ownership root and owning path. +pub(crate) type DeclarationRow = ( + QualifiedRow, + String, + i64, + i64, + String, + RootRow, + Option, +); +/// Consumer, declaration and what the consumer uses it through. +type UsageRow = (QualifiedRow, QualifiedRow, Option); +type IndexRows = (Vec, Vec, Vec); +/// Declaration position, path, scope, ownership root and owning path. +type RelocationRow = (usize, String, &'static str, RootOut, Option); +type RootOut = (String, &'static str, Option<&'static str>); + +fn invalid(what: &str) -> pyo3::PyErr { + PyValueError::new_err(format!("invalid scope placement {what}")) +} + +pub(crate) fn resource_identity((kind, name): IdentityRow) -> PyResult { + Ok(ResourceIdentity { + kind: ResourceKind::parse(&kind).ok_or_else(|| invalid("resource kind"))?, + name, + }) +} + +pub(crate) fn declaration_identity( + (kind, name, owner): QualifiedRow, +) -> PyResult { + Ok(DeclarationIdentity { + kind: DeclarationKind::parse(&kind).ok_or_else(|| invalid("declaration kind"))?, + name, + owner: owner.map(resource_identity).transpose()?, + }) +} + +/// Resource and declaration kinds share no values, so the kind tells the identities apart. +pub(crate) fn qualified_identity(row: QualifiedRow) -> PyResult { + if row.2.is_none() + && let Some(kind) = ResourceKind::parse(&row.0) + { + return Ok(QualifiedIdentity::Resource(ResourceIdentity { + kind, + name: row.1, + })); + } + Ok(QualifiedIdentity::Declaration(declaration_identity(row)?)) +} + +pub(crate) fn ownership_root((path, kind, resource_kind): RootRow) -> PyResult { + Ok(OwnershipRoot { + path, + kind: OwnershipRootKind::parse(&kind).ok_or_else(|| invalid("root kind"))?, + resource_kind: resource_kind + .map(|kind| ResourceKind::parse(&kind).ok_or_else(|| invalid("resource kind"))) + .transpose()?, + }) +} + +fn root_out(root: &OwnershipRoot) -> RootOut { + ( + root.path.clone(), + root.kind.as_str(), + root.resource_kind.map(ResourceKind::as_str), + ) +} + +fn relocation_row(item: Relocation) -> RelocationRow { + ( + item.declaration, + item.path, + item.scope.as_str(), + root_out(&item.ownership_root), + item.owning_path, + ) +} + +pub(crate) fn scope_kind(value: &str) -> PyResult { + ScopeKind::parse(value).ok_or_else(|| invalid("scope")) +} + +/// A resource row as its record. +pub(crate) fn resource_record((identity, path, root): ResourceRow) -> PyResult { + Ok(ResourceRecord { + identity: resource_identity(identity)?, + path, + ownership_root: ownership_root(root)?, + }) +} + +/// A declaration row as its record, without macro dependencies. +pub(crate) fn declaration_record(row: DeclarationRow) -> PyResult { + let (identity, path, line, column, scope, root, owning_path) = row; + Ok(DeclarationRecord { + identity: declaration_identity(identity)?, + path, + line, + column, + scope: scope_kind(&scope)?, + ownership_root: ownership_root(root)?, + owning_path, + dependencies: Vec::new(), + }) +} + +fn placement_index((resources, declarations, usages): IndexRows) -> PyResult { + let mut index: PlacementIndex = PlacementIndex::default(); + for row in resources { + index.resources.push(resource_record(row)?); + } + for row in declarations { + index.declarations.push(declaration_record(row)?); + } + for (consumer, declaration, through) in usages { + index.usages.push(PlacementUsage { + consumer: qualified_identity(consumer)?, + declaration: declaration_identity(declaration)?, + through: through.map(qualified_identity).transpose()?, + }); + } + Ok(index) +} + +/// Each `(declaration position, code, message)` placement diagnostic, in declaration order. +#[pyfunction] +fn scope_placement_diagnostics( + py: Python<'_>, + rows: IndexRows, +) -> PyResult> { + let index: PlacementIndex = placement_index(rows)?; + let diagnostics: Vec = py + .compiler_detach(|| Ok(placement_diagnostics(&index))) + .map_err(compiler_error)?; + Ok(diagnostics + .into_iter() + .map(|item| (item.declaration, item.code.as_str(), item.message)) + .collect()) +} + +/// Each declaration that moves when `resource` moves to `destination`, or `None` if unsettled. +#[pyfunction] +fn scope_relocations( + py: Python<'_>, + rows: IndexRows, + resource: IdentityRow, + destination: &str, +) -> PyResult>> { + let index: PlacementIndex = placement_index(rows)?; + let resource: ResourceIdentity = resource_identity(resource)?; + let relocations: Option> = py + .compiler_detach(|| Ok(relocate_for_resource_move(&index, &resource, destination))) + .map_err(compiler_error)?; + Ok(relocations.map(|items| items.into_iter().map(relocation_row).collect())) +} + +/// The project-relative folder where a declaration of `kind` with this placement lives. +#[pyfunction] +fn scope_declaration_directory(kind: &str, scope: &str, path: Option<&str>) -> PyResult { + let kind: DeclarationKind = + DeclarationKind::parse(kind).ok_or_else(|| invalid("declaration kind"))?; + Ok(declaration_directory(kind, scope_kind(scope)?, path)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scope_placement_diagnostics, module)?)?; + module.add_function(wrap_pyfunction!(scope_relocations, module)?)?; + module.add_function(wrap_pyfunction!(scope_declaration_directory, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs index fbac057e45..0c4cb93e52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/registration.rs @@ -2,11 +2,19 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::scopes::{declaration_contexts, relationship_names, scope_index}; +use crate::bindings::_helpers::scopes::{ + compile_usages, declaration_contexts, placement, relationship_names, scope_cache, scope_index, + scope_lookup, scope_reports, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { scope_index::register(module)?; relationship_names::register(module)?; declaration_contexts::register(module)?; + placement::register(module)?; + compile_usages::register(module)?; + scope_lookup::register(module)?; + scope_cache::register(module)?; + scope_reports::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs new file mode 100644 index 0000000000..64c4749f35 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/relationship_scan.rs @@ -0,0 +1,109 @@ +//! Relationship facts of SQL tests and scenarios, reusing and storing whole files' scans. + +use pyo3::FromPyObject; +use pyo3::prelude::{Bound, Py, PyResult, Python}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_scopes::relationship_names::main::relationship_facts::relationship_facts; +use sqlbuild_scopes::relationship_names::main::stored_expected_names::stored_expected_names; +use sqlbuild_scopes::relationship_names::models::{ + RelationshipScan, ScenarioInput, TestBlockInput, TestFileInput, +}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::cache::sql_test_scan_store::SqlTestScanStore; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::bindings::types::CompilerDetach; + +/// A test file's relative path and each block's name, SQL and mode. +pub(crate) type TestFileRow = (String, Vec<(String, String, String)>); +/// A scenario's name, relative path and SQL. +pub(crate) type ScenarioRow = (String, String, String); + +/// The lexical rules the scans follow and the store, with its key prefix, they reuse. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +pub(crate) struct RelationshipScanSettings { + syntax: LexicalSyntaxInput, + /// The shared scan store, or `None` when this compile keeps no cache. + store: Option>, + /// The scan algorithm and lexical rules keying stored files. + scan_key: (String, String), +} + +/// One file's store key: the key prefix, its path, then each block's SQL and mode. +fn key_parts(settings: &RelationshipScanSettings, file: &TestFileRow) -> Vec { + let (relative_path, blocks) = file; + let mut parts: Vec = vec![ + settings.scan_key.0.clone(), + settings.scan_key.1.clone(), + relative_path.clone(), + ]; + for (_, sql, mode) in blocks { + parts.push(sql.clone()); + parts.push(mode.clone()); + } + parts +} + +/// Every fact and fault, after storing each file that scanned without a fault. +pub(crate) fn scanned_relationships( + py: Python<'_>, + files: &[TestFileRow], + scenarios: &[ScenarioRow], + settings: RelationshipScanSettings, +) -> PyResult { + let store: Option> = + settings.store.as_ref().map(|store| store.bind(py).clone()); + let mut stored: Vec>>> = Vec::with_capacity(files.len()); + for file in files { + stored.push(match &store { + Some(store) => store.get().decoded(key_parts(&settings, file), |value| { + stored_expected_names(value, file.1.len()) + })?, + None => None, + }); + } + let mut inputs: Vec> = Vec::with_capacity(files.len()); + for ((relative_path, blocks), stored) in files.iter().zip(stored) { + let mut block_inputs: Vec> = Vec::with_capacity(blocks.len()); + for (name, sql, mode) in blocks { + block_inputs.push(TestBlockInput { name, sql, mode }); + } + inputs.push(TestFileInput { + relative_path, + blocks: block_inputs, + stored, + }); + } + let scenario_inputs: Vec> = scenarios + .iter() + .map(|(name, relative_path, sql)| ScenarioInput { + name, + relative_path, + sql, + }) + .collect(); + let syntax: LexicalSyntax = settings.syntax.clone().into(); + let python = active_python_text(); + let scan: RelationshipScan = py + .compiler_detach(|| { + Ok(relationship_facts( + &inputs, + &scenario_inputs, + &syntax, + python, + )) + }) + .map_err(compiler_error)?; + if let Some(store) = &store { + for (position, names) in &scan.scanned_files { + let value: Vec = serde_json::to_vec(names) + .map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))?; + store + .get() + .stored(key_parts(&settings, &files[*position]), value)?; + } + } + Ok(scan) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs new file mode 100644 index 0000000000..0efe8fe7cc --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_cache.rs @@ -0,0 +1,418 @@ +//! The persistent scope index cache: fingerprint, verified read and atomic write. + +use std::path::PathBuf; + +use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_scopes::scope_cache::main::encode_scope_cache::encode_scope_cache; +use sqlbuild_scopes::scope_cache::main::read_scope_cache::read_scope_cache; +use sqlbuild_scopes::scope_cache::main::scope_fingerprint::scope_fingerprint; +use sqlbuild_scopes::scope_cache::main::write_scope_cache::write_scope_cache; +use sqlbuild_scopes::scope_cache::models::{ + CachedDeclaration, CachedDiagnostic, CachedInaccessible, CachedIndex, CachedUsage, + CachedVisibility, ConstantFacts, EnumFacts, Fingerprint, FingerprintInputs, MacroFacts, +}; +use sqlbuild_scopes::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, GrantKind, OwnershipRoot, QualifiedIdentity, + ReportGrant, ResourceIdentity, ResourceKind, ResourceRecord, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::scopes::placement::{ + DeclarationRow, IdentityRow, QualifiedRow, ResourceRow, RootRow, declaration_identity, + declaration_record, ownership_root, qualified_identity, resource_identity, resource_record, +}; +use crate::bindings::types::CompilerDetach; + +/// Parameters, dependencies and source digest of a macro. +type MacroRow = (Vec, Vec, String); +/// Member names and scalar type of an enum. +type EnumRow = (Vec, String); +/// Logical type, collection kind, item count, nullability and rendering of a constant. +type ConstantRow = (String, Option, Option, bool, Option); +type FullDeclarationRow = ( + DeclarationRow, + Option, + Option, + Option, +); +type UsageRow = ( + QualifiedRow, + QualifiedRow, + String, + Option, + Option, +); +type GrantRow = (IdentityRow, QualifiedRow, QualifiedRow, String); +type VisibilityRow = (IdentityRow, QualifiedRow, String, Option); +type InaccessibleRow = (IdentityRow, QualifiedRow, String); +type DiagnosticRow = ( + String, + String, + String, + Option, + Option, + Option, + Option, + Option, +); +type Completeness = (bool, bool, bool, bool, bool, bool); +/// Every field of a full scope index as rows. +type IndexRows = ( + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Completeness, +); +/// Config files, candidate roots, declaration folders, named declaration folders, version. +type FingerprintRow = (Vec, Vec, Vec, Vec, String); + +fn invalid(what: &str) -> pyo3::PyErr { + PyValueError::new_err(format!("invalid scope cache {what}")) +} + +fn grant_kind(value: &str) -> PyResult { + [GrantKind::ExpectedModel, GrantKind::TestedMacro] + .into_iter() + .find(|kind| kind.as_str() == value) + .ok_or_else(|| invalid("grant kind")) +} + +fn optional_identity(row: Option) -> PyResult> { + row.map(qualified_identity).transpose() +} + +fn full_declaration(row: FullDeclarationRow) -> PyResult { + let (record, macro_row, enum_row, constant_row) = row; + let macro_facts: Option = match macro_row { + None => None, + Some((parameters, dependencies, source_digest)) => Some(MacroFacts { + parameters, + dependencies: dependencies + .into_iter() + .map(declaration_identity) + .collect::>()?, + source_digest, + }), + }; + Ok(CachedDeclaration { + record: declaration_record(record)?, + macro_facts, + enum_facts: enum_row.map(|(members, scalar_type)| EnumFacts { + members, + scalar_type, + }), + constant_facts: constant_row.map( + |(logical_type, collection_kind, item_count, nullable, render_as)| ConstantFacts { + logical_type, + collection_kind, + item_count, + nullable, + render_as, + }, + ), + }) +} + +fn cached_index(rows: IndexRows) -> PyResult { + let ( + roots, + resources, + declarations, + usages, + grants, + visibility, + inaccessible, + diagnostics, + done, + ) = rows; + let mut index: CachedIndex = CachedIndex { + completeness: [done.0, done.1, done.2, done.3, done.4, done.5], + ..CachedIndex::default() + }; + for row in roots { + index.roots.push(ownership_root(row)?); + } + for row in resources { + index.resources.push(resource_record(row)?); + } + for row in declarations { + index.declarations.push(full_declaration(row)?); + } + for (consumer, declaration, kind, through, enum_member) in usages { + index.usages.push(CachedUsage { + consumer: qualified_identity(consumer)?, + declaration: declaration_identity(declaration)?, + kind, + through: optional_identity(through)?, + enum_member, + }); + } + for (resource, declaration, through, kind) in grants { + index.grants.push(ReportGrant { + resource: resource_identity(resource)?, + declaration: declaration_identity(declaration)?, + through: qualified_identity(through)?, + kind: grant_kind(&kind)?, + }); + } + for (resource, declaration, reason, through) in visibility { + index.visibility.push(CachedVisibility { + resource: resource_identity(resource)?, + declaration: declaration_identity(declaration)?, + reason, + through: optional_identity(through)?, + }); + } + for (resource, declaration, reason) in inaccessible { + index.inaccessible.push(CachedInaccessible { + resource: resource_identity(resource)?, + declaration: declaration_identity(declaration)?, + reason, + }); + } + for (code, message, severity, path, line, column, declaration, resource) in diagnostics { + index.diagnostics.push(CachedDiagnostic { + code, + message, + severity, + path, + line, + column, + declaration: declaration.map(declaration_identity).transpose()?, + resource: resource.map(resource_identity).transpose()?, + }); + } + Ok(index) +} + +fn identity_row(identity: &ResourceIdentity) -> IdentityRow { + (identity.kind.as_str().to_owned(), identity.name.clone()) +} + +fn declaration_row(identity: &DeclarationIdentity) -> QualifiedRow { + ( + identity.kind.as_str().to_owned(), + identity.name.clone(), + identity.owner.as_ref().map(identity_row), + ) +} + +fn qualified_row(identity: &QualifiedIdentity) -> QualifiedRow { + match identity { + QualifiedIdentity::Resource(identity) => ( + identity.kind.as_str().to_owned(), + identity.name.clone(), + None, + ), + QualifiedIdentity::Declaration(identity) => declaration_row(identity), + } +} + +fn root_row(root: &OwnershipRoot) -> RootRow { + ( + root.path.clone(), + root.kind.as_str().to_owned(), + root.resource_kind + .map(|kind| ResourceKind::as_str(kind).to_owned()), + ) +} + +fn resource_row(record: &ResourceRecord) -> ResourceRow { + ( + identity_row(&record.identity), + record.path.clone(), + root_row(&record.ownership_root), + ) +} + +fn record_row(record: &DeclarationRecord) -> DeclarationRow { + ( + declaration_row(&record.identity), + record.path.clone(), + record.line, + record.column, + record.scope.as_str().to_owned(), + root_row(&record.ownership_root), + record.owning_path.clone(), + ) +} + +fn full_declaration_row(item: &CachedDeclaration) -> FullDeclarationRow { + ( + record_row(&item.record), + item.macro_facts.as_ref().map(|facts| { + ( + facts.parameters.clone(), + facts.dependencies.iter().map(declaration_row).collect(), + facts.source_digest.clone(), + ) + }), + item.enum_facts + .as_ref() + .map(|facts| (facts.members.clone(), facts.scalar_type.clone())), + item.constant_facts.as_ref().map(|facts| { + ( + facts.logical_type.clone(), + facts.collection_kind.clone(), + facts.item_count, + facts.nullable, + facts.render_as.clone(), + ) + }), + ) +} + +fn index_rows(index: &CachedIndex) -> IndexRows { + let done: [bool; 6] = index.completeness; + ( + index.roots.iter().map(root_row).collect(), + index.resources.iter().map(resource_row).collect(), + index + .declarations + .iter() + .map(full_declaration_row) + .collect(), + index + .usages + .iter() + .map(|item| { + ( + qualified_row(&item.consumer), + declaration_row(&item.declaration), + item.kind.clone(), + item.through.as_ref().map(qualified_row), + item.enum_member.clone(), + ) + }) + .collect(), + index + .grants + .iter() + .map(|item| { + ( + identity_row(&item.resource), + declaration_row(&item.declaration), + qualified_row(&item.through), + item.kind.as_str().to_owned(), + ) + }) + .collect(), + index + .visibility + .iter() + .map(|item| { + ( + identity_row(&item.resource), + declaration_row(&item.declaration), + item.reason.clone(), + item.through.as_ref().map(qualified_row), + ) + }) + .collect(), + index + .inaccessible + .iter() + .map(|item| { + ( + identity_row(&item.resource), + declaration_row(&item.declaration), + item.reason.clone(), + ) + }) + .collect(), + index + .diagnostics + .iter() + .map(|item| { + ( + item.code.clone(), + item.message.clone(), + item.severity.clone(), + item.path.clone(), + item.line, + item.column, + item.declaration.as_ref().map(declaration_row), + item.resource.as_ref().map(identity_row), + ) + }) + .collect(), + (done[0], done[1], done[2], done[3], done[4], done[5]), + ) +} + +/// The scope index fingerprint of `project_dir`, `None` when it cannot be read. +#[pyfunction] +fn scope_index_fingerprint( + py: Python<'_>, + project_dir: PathBuf, + inputs: FingerprintRow, +) -> PyResult> { + let (config_files, roots, declaration_directories, named_declaration_directories, version) = + inputs; + let inputs: FingerprintInputs = FingerprintInputs { + config_files, + roots, + declaration_directories, + named_declaration_directories, + sqlbuild_version: version, + }; + let fingerprint: Fingerprint = py + .compiler_detach(|| Ok(scope_fingerprint(&project_dir, &inputs))) + .map_err(compiler_error)?; + match fingerprint { + Fingerprint::Hash(hash) => Ok(Some(hash)), + Fingerprint::Unavailable => Ok(None), + Fingerprint::Unserializable => Err(PyTypeError::new_err( + "a scope-relevant config value is not JSON serializable", + )), + } +} + +/// The verified cached index rows at `path` for `fingerprint`, else `None`. +#[pyfunction] +fn read_scope_index_cache( + py: Python<'_>, + path: PathBuf, + fingerprint: String, +) -> PyResult> { + let cached = py + .compiler_detach(|| Ok(read_scope_cache(&path, &fingerprint))) + .map_err(compiler_error)?; + Ok(match cached { + Ok(index) => Some(index_rows(&index)), + Err(_) => None, + }) +} + +/// Atomically write the canonical index `rows` for `fingerprint`; whether the write landed. +#[pyfunction] +fn write_scope_index_cache( + py: Python<'_>, + location: (PathBuf, String), + fingerprint: String, + rows: IndexRows, +) -> PyResult { + let index: CachedIndex = cached_index(rows)?; + let (directory, file_name) = location; + py.compiler_detach(|| { + Ok(match encode_scope_cache(&index, &fingerprint) { + Ok(envelope) => matches!(write_scope_cache(&directory, &file_name, &envelope), Ok(())), + Err(_) => false, + }) + }) + .map_err(compiler_error) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scope_index_fingerprint, module)?)?; + module.add_function(wrap_pyfunction!(read_scope_index_cache, module)?)?; + module.add_function(wrap_pyfunction!(write_scope_index_cache, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index 4df342b0ac..271d118fb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -4,18 +4,21 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{IntoPyObject, pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_scopes::scope_index::main::build_scope_index::build_scope_index; use sqlbuild_scopes::scope_index::main::relationship_grants::relationship_grants; -use sqlbuild_scopes::scope_index::main::scope_lookup::scope_lookup_groups; use sqlbuild_scopes::scope_index::models::{ DeclarationIdentity, DeclarationInput, DeclarationRecord, GrantRecord, GrantThrough, - KeyedGroups, RelationshipFact, ResourceIdentity, ResourceInput, ScopeDeferral, ScopeDiagnostic, - ScopeIndex, ScopeInputs, ScopeLookupGroups, + RelationshipFact, ResourceIdentity, ResourceInput, ScopeDeferral, ScopeDiagnostic, ScopeIndex, + ScopeInputs, }; use sqlbuild_scopes::scope_index::models::{ DeclarationKind, ResourceKind, ResourceRoot, ScopeKind, }; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::scopes::relationship_scan::{ + RelationshipScanSettings, ScenarioRow, TestFileRow, scanned_relationships, +}; use crate::bindings::types::CompilerDetach; +use sqlbuild_scopes::relationship_names::models::RelationshipScan; type IdentityRow = (String, String); type DeclarationIdentityRow = (String, String, Option); @@ -32,7 +35,6 @@ type DeclarationRow = ( Option, Vec, ); -type FactRow = (String, String, Vec, Vec, Vec); type DeclarationRecordRow = ( String, String, @@ -59,28 +61,8 @@ type IndexRows = ( bool, ); type GrantRow = (&'static str, String, usize, ThroughRow, &'static str); -type GroupRows = (Vec>, bool); -type VisibilityRows = ( - Vec, - Vec<(IdentityRow, Vec)>, - Vec<(String, Vec)>, - Vec<(String, Vec)>, -); -type LookupRows = ( - Vec, - Vec, - Vec, - Vec, - ( - GroupRows, - GroupRows, - GroupRows, - GroupRows, - GroupRows, - GroupRows, - ), - VisibilityRows, -); +/// The grants, or `None` when the index cannot place them, and each `(path, message)` fault. +type GrantedRelationships = (Option>, Vec<(String, String)>); /// What a grant came through: an expected model's name or a tested macro's declaration index. #[derive(IntoPyObject)] @@ -120,64 +102,26 @@ impl NativeScopeIndex { ) } - /// Resolve and keep the relationship grants of `facts`; `None` when Python must run. - fn grant(&mut self, py: Python<'_>, facts: Vec) -> PyResult>> { - let Some(facts) = facts - .into_iter() - .map(relationship_fact) - .collect::>>() - else { - return Ok(None); - }; + /// Read every test's and scenario's relationship facts, then resolve and keep their grants. + fn grant_relationships( + &mut self, + py: Python<'_>, + files: Vec, + scenarios: Vec, + settings: RelationshipScanSettings, + ) -> PyResult { + let scan: RelationshipScan = scanned_relationships(py, &files, &scenarios, settings)?; let index: &ScopeIndex = &self.index; + let facts: &[RelationshipFact] = &scan.facts; let grants: Result, ScopeDeferral> = py - .compiler_detach(|| Ok(relationship_grants(index, &facts))) + .compiler_detach(|| Ok(relationship_grants(index, facts))) .map_err(compiler_error)?; let Ok(grants) = grants else { - return Ok(None); + return Ok((None, scan.faults)); }; let rows: Vec = grants.iter().map(grant_row).collect(); self.grants = grants; - Ok(Some(rows)) - } - - /// The lookup orders and groups of the index with its kept grants; `None` when Python must run. - fn lookup(&self, py: Python<'_>) -> PyResult> { - let index: &ScopeIndex = &self.index; - let grants: &[GrantRecord] = &self.grants; - let groups: Result = py - .compiler_detach(|| Ok(scope_lookup_groups(index, grants))) - .map_err(compiler_error)?; - let Ok(groups) = groups else { - return Ok(None); - }; - Ok(Some(( - index.canonical_resources.clone(), - index.canonical_declarations.clone(), - groups.canonical_usages, - groups.canonical_grants, - ( - group_rows(groups.resources), - group_rows(groups.resources_by_path), - group_rows(groups.declarations), - group_rows(groups.usages_by_consumer), - group_rows(groups.usages_by_declaration), - group_rows(groups.grants_by_resource), - ), - ( - groups.visibility.global, - groups - .visibility - .private - .into_iter() - .map(|(owner, positions)| { - ((owner.kind.as_str().to_owned(), owner.name), positions) - }) - .collect(), - groups.visibility.local, - groups.visibility.inherited, - ), - ))) + Ok((Some(rows), scan.faults)) } } @@ -278,17 +222,6 @@ fn declaration_identity( }) } -fn relationship_fact( - (kind, name, expected_models, called_macros, tested_macros): FactRow, -) -> Option { - Some(RelationshipFact { - resource: resource_identity((kind, name))?, - expected_models, - called_macros, - tested_macros, - }) -} - fn declaration_row(record: &DeclarationRecord) -> DeclarationRecordRow { ( record.path.clone(), @@ -327,10 +260,6 @@ fn grant_row(grant: &GrantRecord) -> GrantRow { ) } -fn group_rows(groups: KeyedGroups) -> GroupRows { - (groups.groups, groups.repr_ordered) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(build_native_scope_index, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs new file mode 100644 index 0000000000..89b1c294b4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -0,0 +1,345 @@ +//! The scope lookup of a full scope index: canonical orders, mappings, targets and visibility. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use sqlbuild_scopes::scope_index::main::normalize_scope_path::normalize_scope_path; +use sqlbuild_scopes::scope_index::main::report_lookup::report_lookup; +use sqlbuild_scopes::scope_index::main::report_target::report_target; +use sqlbuild_scopes::scope_index::main::resolve_report_visibility::resolve_report_visibility; +use sqlbuild_scopes::scope_index::models::{ + GrantKind, IdentityFailure, KeyedGroups, RejectedPath, ReportDiagnosticKey, ReportGrant, + ReportLookup, ReportRecords, ReportTarget, ReportUsage, ResolvedVisibility, ResolvedVisible, + ScopePathFailure, TargetFailure, TargetMatch, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::scopes::placement::{ + DeclarationRow, IdentityRow, QualifiedRow, ResourceRow, RootRow, declaration_identity, + declaration_record, ownership_root, qualified_identity, resource_identity, resource_record, +}; +use crate::bindings::types::CompilerDetach; + +/// Consumer, declaration, usage kind and what the consumer uses it through. +type UsageRow = (QualifiedRow, QualifiedRow, String, Option); +/// Resource, declaration, what it came through, and the grant kind. +type GrantRow = (IdentityRow, QualifiedRow, QualifiedRow, String); +/// Path, line, column and code of a diagnostic. +type DiagnosticKeyRow = (Option, Option, Option, String); +/// Every record of a full scope index as rows. +type RecordRows = ( + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, +); +type GroupRows = Vec>; +/// Roots, resources, declarations, usages, grants and diagnostics in canonical order. +type OrderRows = ( + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, +); +/// Each mapping's groups, then every input resource's normalized path key. +type MappingRows = ( + GroupRows, + GroupRows, + GroupRows, + GroupRows, + GroupRows, + GroupRows, + Vec, +); +/// Global, private, local and inherited canonical declaration positions. +type VisibilityRows = ( + Vec, + Vec<(IdentityRow, Vec)>, + Vec<(String, Vec)>, + Vec<(String, Vec)>, +); +/// The canonical orders, mappings and visibility groups, else the rejected path's message. +type LookupRows = ( + Option<(OrderRows, MappingRows, VisibilityRows)>, + Option, +); +/// `(declarations, canonical positions)` matched, else identity and path error messages. +type TargetRows = (bool, Vec, Option, Option); +/// Visible `(position, reason, grant)`, inaccessible `(position, reason)`, else the path error. +type ResolvedRows = ( + Vec<(usize, &'static str, Option)>, + Vec<(usize, &'static str)>, + Option, +); + +/// One full scope index's records and lookup, kept for targets and classification. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeScopeLookup { + pub(crate) records: ReportRecords, + pub(crate) lookup: ReportLookup, + /// Each input resource, declaration and grant's canonical position. + canonical: CanonicalPositions, +} + +/// The inverse of the lookup's canonical orders. +struct CanonicalPositions { + resources: Vec, + declarations: Vec, + grants: Vec, +} + +fn inverse(order: &[usize]) -> Vec { + let mut positions: Vec = vec![0; order.len()]; + for (canonical, input) in order.iter().enumerate() { + positions[*input] = canonical; + } + positions +} + +pub(crate) fn grant_kind(value: &str) -> PyResult { + match value { + "expected_model" => Ok(GrantKind::ExpectedModel), + "tested_macro" => Ok(GrantKind::TestedMacro), + _ => Err(pyo3::exceptions::PyValueError::new_err( + "invalid scope grant kind", + )), + } +} + +fn report_records(rows: RecordRows) -> PyResult { + let (roots, resources, declarations, usages, grants, diagnostics) = rows; + let mut records: ReportRecords = ReportRecords::default(); + for root in roots { + records.roots.push(ownership_root(root)?); + } + for row in resources { + records.resources.push(resource_record(row)?); + } + for row in declarations { + records.declarations.push(declaration_record(row)?); + } + for (consumer, declaration, kind, through) in usages { + records.usages.push(ReportUsage { + consumer: qualified_identity(consumer)?, + declaration: declaration_identity(declaration)?, + kind, + through: through.map(qualified_identity).transpose()?, + }); + } + for (resource, declaration, through, kind) in grants { + records.grants.push(ReportGrant { + resource: resource_identity(resource)?, + declaration: declaration_identity(declaration)?, + through: qualified_identity(through)?, + kind: grant_kind(&kind)?, + }); + } + for (path, line, column, code) in diagnostics { + records.diagnostics.push(ReportDiagnosticKey { + path, + line, + column, + code, + }); + } + Ok(records) +} + +/// Python's `InvalidScopePathError` message for a rejected path. +pub(crate) fn path_message(rejected: &RejectedPath) -> String { + match rejected.failure { + ScopePathFailure::NotProjectRelative => { + format!("Scope path must be project-relative: {}", rejected.path) + } + ScopePathFailure::EscapesProject => { + format!("Scope path escapes the project: {}", rejected.path) + } + } +} + +/// Python's `InvalidQualifiedIdentityError` message for a rejected identity. +pub(crate) fn identity_message(failure: &IdentityFailure, value: &str) -> String { + let shown = |text: &str| -> String { python_str_repr(active_python_text(), text) }; + match failure { + IdentityFailure::NoName => format!("Qualified identity has no name: {}", shown(value)), + IdentityFailure::UnknownKind(kind) => { + format!("Unknown qualified identity kind: {}", shown(kind)) + } + IdentityFailure::InvalidPrivate => format!("Invalid private identity: {}", shown(value)), + IdentityFailure::Unqualified => format!( + "Expected a kind-qualified identity such as 'model:name': {}", + shown(value) + ), + } +} + +fn group_rows(groups: KeyedGroups) -> GroupRows { + groups.groups +} + +fn lookup_rows(lookup: &ReportLookup) -> (OrderRows, MappingRows, VisibilityRows) { + let groups = lookup.groups.clone(); + let visibility = lookup.visibility.clone(); + ( + ( + lookup.roots.clone(), + lookup.resources.clone(), + lookup.declarations.clone(), + lookup.usages.clone(), + lookup.grants.clone(), + lookup.diagnostics.clone(), + ), + ( + group_rows(groups.resources), + group_rows(groups.resources_by_path), + group_rows(groups.declarations), + group_rows(groups.usages_by_consumer), + group_rows(groups.usages_by_declaration), + group_rows(groups.grants_by_resource), + lookup.resource_paths.clone(), + ), + ( + visibility.global, + visibility + .private + .into_iter() + .map(|(owner, positions)| ((owner.kind.as_str().to_owned(), owner.name), positions)) + .collect(), + visibility.local, + visibility.inherited, + ), + ) +} + +/// Input positions as canonical positions. +fn canonical(order: &[usize], positions: Vec) -> Vec { + positions + .into_iter() + .map(|position| order[position]) + .collect() +} + +#[pymethods] +impl NativeScopeLookup { + /// The records `target` names: an identity row, else qualified identity or path text. + fn target(&self, identity: Option, text: &str) -> PyResult { + let target: ReportTarget = match identity { + Some(row) => ReportTarget::Identity(qualified_identity(row)?), + None => ReportTarget::Text(text.to_owned()), + }; + Ok(match report_target(&self.records, &self.lookup, &target) { + Ok(TargetMatch::Resources(positions)) => ( + false, + canonical(&self.canonical.resources, positions), + None, + None, + ), + Ok(TargetMatch::Declarations(positions)) => ( + true, + canonical(&self.canonical.declarations, positions), + None, + None, + ), + Err(TargetFailure::Identity(failure)) => ( + false, + Vec::new(), + Some(identity_message(&failure, text)), + None, + ), + Err(TargetFailure::Path(rejected)) => { + (false, Vec::new(), None, Some(path_message(&rejected))) + } + }) + } + + /// Python's `_classify_resource` of one resource, with inaccessible reasons. + fn resolve(&self, py: Python<'_>, resource: ResourceRow) -> PyResult { + let resource = resource_record(resource)?; + let (records, lookup) = (&self.records, &self.lookup); + let resolved = py + .compiler_detach(|| Ok(resolve_report_visibility(records, lookup, &resource))) + .map_err(compiler_error)?; + Ok(match resolved { + Ok(resolved) => resolved_rows(resolved, &self.canonical.grants), + Err(rejected) => (Vec::new(), Vec::new(), Some(path_message(&rejected))), + }) + } +} + +fn visible_row(entry: ResolvedVisible, grants: &[usize]) -> (usize, &'static str, Option) { + let grant: Option = entry.grant.map(|grant| grants[grant]); + (entry.position, entry.reason.as_str(), grant) +} + +fn resolved_rows(resolved: ResolvedVisibility, grants: &[usize]) -> ResolvedRows { + let visible: Vec<(usize, &'static str, Option)> = resolved + .visible + .into_iter() + .map(|entry| visible_row(entry, grants)) + .collect(); + ( + visible, + resolved + .inaccessible + .into_iter() + .map(|(position, reason)| (position, reason.as_str())) + .collect(), + None, + ) +} + +/// The lookup of a full scope index's records, else the message of the path Python rejects. +#[pyfunction] +fn build_native_scope_lookup( + py: Python<'_>, + rows: RecordRows, +) -> PyResult<(Option, LookupRows)> { + let records: ReportRecords = report_records(rows)?; + let built = py + .compiler_detach(|| Ok(report_lookup(&records))) + .map_err(compiler_error)?; + Ok(match built { + Ok(lookup) => { + let rows: (OrderRows, MappingRows, VisibilityRows) = lookup_rows(&lookup); + let canonical: CanonicalPositions = CanonicalPositions { + resources: inverse(&lookup.resources), + declarations: inverse(&lookup.declarations), + grants: inverse(&lookup.grants), + }; + let native: NativeScopeLookup = NativeScopeLookup { + records, + lookup, + canonical, + }; + (Some(native), (Some(rows), None)) + } + Err(rejected) => (None, (None, Some(path_message(&rejected)))), + }) +} + +/// Python's scope `normalize_path`, else the `InvalidScopePathError` message. +#[pyfunction] +fn scope_normalize_path(path: &str) -> (Option, Option) { + match normalize_scope_path(path) { + Ok(normalized) => (Some(normalized), None), + Err(failure) => ( + None, + Some(path_message(&RejectedPath { + failure, + path: path.to_owned(), + })), + ), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_native_scope_lookup, module)?)?; + module.add_function(wrap_pyfunction!(scope_normalize_path, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_rows.rs new file mode 100644 index 0000000000..584e61190a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_rows.rs @@ -0,0 +1,741 @@ +//! Rows of scope report requests and results, in both directions across the boundary. + +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple, + PyTypeMethods, +}; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; +use sqlbuild_scopes::scope_index::main::format_identity::format_identity; +use sqlbuild_scopes::scope_index::models::QualifiedIdentity; +use sqlbuild_scopes::scope_report::models::{ + BrowseResult, Completeness, ConstantValues, DeclarationReport, DeclarationTarget, + DeclarationValues, DiagnosticFacts, EnumValues, Explanation, ListResult, MacroValues, + MovePreview, Provenance, QueryTarget, ReportDiagnostic, ReportFilters, ResourceReport, + ScopeChainEntry, ScopeFolder, ScopeReport, ScopeSection, SourceLocation, +}; + +use crate::bindings::_helpers::scopes::placement::{ + IdentityRow, QualifiedRow, declaration_identity, qualified_identity, resource_identity, +}; + +type PyObject = Py; + +/// A query target: an identity row, else text, and whether the text came from a `PurePath`. +pub(crate) type TargetRow = (Option, String, bool); +/// Every `ScopeReportFilters` field in order. +pub(crate) type FiltersRow = ( + Vec, + bool, + Option, + Vec, + Option, + bool, + i64, + String, + Option, + i64, + i64, + String, +); +/// Parameters, dependencies and source digest. +type MacroRow = (Vec, Vec, String); +/// Member names and scalar type. +type EnumRow = (Vec, String); +/// Logical type, collection kind, item count, nullable and render-as. +type ConstantRow = (String, Option, Option, bool, Option); +pub(crate) type ValuesRow = (Option, Option, Option); +pub(crate) type CompletenessRow = (bool, bool, bool, bool, bool, bool); +/// Code, message, severity, path, line, column, declaration and resource identities. +pub(crate) type DiagnosticInRow = ( + String, + String, + String, + Option, + Option, + Option, + Option, + Option, +); +/// An index diagnostic's canonical position, else a raised `(code, message, path)`. +type DiagnosticOutRow = (Option, Option<(String, String, Option)>); +/// Identity, kind, name, owner, definition, scope and owning path. +type HeadRow = ( + String, + String, + String, + Option, + (String, i64, i64), + String, + Option, +); +/// Visibility, inaccessible reason and metadata entries. +type StandingRow = ( + Option<(String, Option)>, + Option, + Vec<(String, M)>, +); +/// Consumers, dependencies, grants, required scope, required path and promotion impact. +type LinksRow = ( + Vec, + Vec, + Vec, + Option, + Option, + Vec, +); +pub(crate) type DeclarationRow = (HeadRow, StandingRow, LinksRow); +type SectionRow = ( + String, + i64, + i64, + bool, + i64, + bool, + bool, + Option, + Option, + i64, +); +type ResourceReportRow = (String, Option, Option, bool, bool, i64); +type ChainRow = (String, String, i64); +type ExplanationRow = (Option>, bool); +/// Every `MovePreview` field in order. +type MoveRow = ( + String, + String, + Option, + Vec>, + Vec>, + Vec>, + Vec, + Vec>, + Vec>, + bool, +); +type FolderRow = (String, String, i64, i64, i64, Vec<(String, i64)>); +/// Every `ScopeReport` field in order. +pub(crate) type ReportRow = ( + ResourceReportRow, + Vec, + Vec>, + Vec>, + Vec>, + Vec>, + FiltersRow, + Vec, + Option>, + Option>, + Vec, + bool, +); +pub(crate) type BrowseRow = (String, Vec, Vec, bool); +pub(crate) type ListRow = (String, Vec>, SectionRow, Vec); +pub(crate) type ExplanationOutRow = (ExplanationRow, Vec); +pub(crate) type MoveOutRow = (Option>, Vec); +pub(crate) type ReportOutRow = ReportRow; +pub(crate) type BrowseOutRow = BrowseRow; +pub(crate) type ListOutRow = ListRow; + +pub(crate) fn target(row: TargetRow) -> PyResult { + let (identity, text, path) = row; + Ok(match (identity, path) { + (Some(identity), _) => QueryTarget::Identity(qualified_identity(identity)?), + (None, true) => QueryTarget::Path(text), + (None, false) => QueryTarget::Text(text), + }) +} + +pub(crate) fn declaration_target( + (identity, text): (Option, String), +) -> PyResult { + Ok(match identity { + Some(identity) => DeclarationTarget::Identity(declaration_identity(identity)?), + None => DeclarationTarget::Text(text), + }) +} + +pub(crate) fn filters(row: FiltersRow) -> ReportFilters { + let ( + sections, + include_nearby, + defined_under, + kinds, + glob, + used_only, + dependency_depth, + sort, + cursor, + page_size, + nearby_depth, + globals, + ) = row; + ReportFilters { + sections, + include_nearby, + defined_under, + kinds, + glob, + used_only, + dependency_depth, + sort, + cursor, + page_size, + nearby_depth, + globals, + } +} + +fn filters_row(filters: &ReportFilters) -> FiltersRow { + ( + filters.sections.clone(), + filters.include_nearby, + filters.defined_under.clone(), + filters.kinds.clone(), + filters.glob.clone(), + filters.used_only, + filters.dependency_depth, + filters.sort.clone(), + filters.cursor.clone(), + filters.page_size, + filters.nearby_depth, + filters.globals.clone(), + ) +} + +pub(crate) fn declaration_values(row: ValuesRow) -> PyResult { + let (macro_row, enum_row, constant_row) = row; + Ok(DeclarationValues { + macro_values: match macro_row { + Some((parameters, dependencies, source_digest)) => Some(MacroValues { + parameters, + dependencies: dependencies + .into_iter() + .map(declaration_identity) + .collect::>()?, + source_digest, + }), + None => None, + }, + enum_values: enum_row.map(|(members, scalar_type)| EnumValues { + members, + scalar_type, + }), + constant_values: constant_row.map( + |(logical_type, collection_kind, item_count, nullable, render_as)| ConstantValues { + logical_type, + collection_kind, + item_count, + nullable, + render_as, + }, + ), + }) +} + +pub(crate) fn completeness(row: CompletenessRow) -> Completeness { + let (discovery, static_visibility, runtime_usage, relationships, placement, promotion_impact) = + row; + Completeness { + discovery, + static_visibility, + runtime_usage, + relationships, + placement, + promotion_impact, + } +} + +/// A diagnostic row's facts, with its identities as qualified text. +pub(crate) fn diagnostic_facts(row: DiagnosticInRow) -> PyResult { + let (code, message, severity, path, line, column, declaration, resource) = row; + Ok(DiagnosticFacts { + code, + message, + severity, + path, + line, + column, + declaration: match declaration { + Some(row) => Some(format_identity(&QualifiedIdentity::Declaration( + declaration_identity(row)?, + ))), + None => None, + }, + resource: match resource { + Some(row) => Some(format_identity(&QualifiedIdentity::Resource( + resource_identity(row)?, + ))), + None => None, + }, + }) +} + +fn diagnostic_out(diagnostic: ReportDiagnostic) -> DiagnosticOutRow { + match diagnostic { + ReportDiagnostic::Index(position) => (Some(position), None), + ReportDiagnostic::Raised(facts) => (None, Some((facts.code, facts.message, facts.path))), + } +} + +pub(crate) fn raised_out(diagnostics: Vec) -> Vec { + diagnostics + .into_iter() + .map(|facts| diagnostic_out(ReportDiagnostic::Raised(facts))) + .collect() +} + +/// A JSON value as Python builds it; arrays become tuples or lists. +pub(crate) fn json_to_python(py: Python<'_>, value: JsonValue, tuples: bool) -> PyResult { + match value { + JsonValue::Null => Ok(py.None()), + JsonValue::Bool(flag) => flag.into_py_any(py), + JsonValue::Integer(integer) => match integer.to_i128() { + Some(number) => number.into_py_any(py), + None => Ok(py + .import("builtins")? + .getattr("int")? + .call1((integer.as_str(),))? + .unbind()), + }, + JsonValue::Float(number) => number.into_py_any(py), + JsonValue::String(text) => text.into_py_any(py), + JsonValue::Utf16(units) => String::from_utf16_lossy(&units).into_py_any(py), + JsonValue::Array(items) => { + let values: Vec = items + .into_iter() + .map(|item| json_to_python(py, item, tuples)) + .collect::>()?; + if tuples { + PyTuple::new(py, values)?.into_py_any(py) + } else { + PyList::new(py, values)?.into_py_any(py) + } + } + JsonValue::Object(entries) => { + let dictionary: Bound<'_, PyDict> = PyDict::new(py); + for (key, item) in entries { + dictionary.set_item(key, json_to_python(py, item, tuples)?)?; + } + dictionary.into_py_any(py) + } + } +} + +/// `_value` of one metadata value; an unsupported type is the error Python's serializer raises. +pub(crate) fn python_to_json(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(flag.is_true())); + } + let py: Python<'_> = value.py(); + if py + .import("enum")? + .getattr("Enum") + .and_then(|enum_type| value.is_instance(&enum_type))? + { + return python_to_json(&value.getattr("value")?); + } + if value.is_instance_of::() { + let decimal: String = value.str()?.extract()?; + return JsonInteger::parse(&decimal) + .map(JsonValue::Integer) + .ok_or_else(|| PyValueError::new_err("Unsupported scope report value: int")); + } + if value.is_instance_of::() { + return Ok(JsonValue::Float(value.extract::()?)); + } + if value.is_instance_of::() { + return Ok(JsonValue::String(value.extract()?)); + } + let pure_path = py.import("pathlib")?.getattr("PurePath")?; + if value.is_instance(&pure_path)? { + return Ok(JsonValue::String( + value.call_method0("as_posix")?.extract()?, + )); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(python_to_json(&item?)?); + } + return Ok(JsonValue::Array(items)); + } + if let Ok(mapping) = value.downcast::() { + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for (key, item) in mapping.iter() { + entries.push((key.str()?.extract()?, python_to_json(&item)?)); + } + return Ok(JsonValue::Object(entries)); + } + let name: String = value.get_type().name()?.extract()?; + Err(PyValueError::new_err(format!( + "Unsupported scope report value: {name}" + ))) +} + +/// `{str(key): value}` with later duplicate keys replacing earlier values in place. +fn unique_entries(entries: Vec<(String, JsonValue)>) -> Vec<(String, JsonValue)> { + let mut unique: Vec<(String, JsonValue)> = Vec::with_capacity(entries.len()); + for (key, value) in entries { + match unique.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => unique.push((key, value)), + } + } + unique +} + +pub(crate) fn declaration_out( + py: Python<'_>, + report: DeclarationReport, +) -> PyResult> { + let metadata: Vec<(String, PyObject)> = report + .metadata + .into_iter() + .map(|(key, value)| Ok((key, json_to_python(py, value, true)?))) + .collect::>()?; + Ok(( + ( + report.identity, + report.kind, + report.name, + report.owner, + ( + report.definition.path, + report.definition.line, + report.definition.column, + ), + report.scope, + report.owning_path, + ), + ( + report + .visibility + .map(|provenance| (provenance.reason, provenance.through)), + report.inaccessible_reason, + metadata, + ), + ( + report.consumers, + report.dependencies, + report.grants, + report.required_scope, + report.required_path, + report.promotion_impact, + ), + )) +} + +pub(crate) fn declaration_in(row: DeclarationRow>) -> PyResult { + let ( + (identity, kind, name, owner, (path, line, column), scope, owning_path), + (visibility, inaccessible_reason, metadata), + (consumers, dependencies, grants, required_scope, required_path, promotion_impact), + ) = row; + let metadata: Vec<(String, JsonValue)> = metadata + .into_iter() + .map(|(key, value)| Ok((key, python_to_json(&value)?))) + .collect::>()?; + Ok(DeclarationReport { + identity, + kind, + name, + owner, + definition: SourceLocation { path, line, column }, + scope, + owning_path, + visibility: visibility.map(|(reason, through)| Provenance { reason, through }), + inaccessible_reason, + metadata: unique_entries(metadata), + consumers, + dependencies, + grants, + required_scope, + required_path, + promotion_impact, + }) +} + +fn declarations_out( + py: Python<'_>, + reports: Vec, +) -> PyResult>> { + reports + .into_iter() + .map(|report| declaration_out(py, report)) + .collect() +} + +fn declarations_in( + rows: Vec>>, +) -> PyResult> { + rows.into_iter().map(declaration_in).collect() +} + +fn section_row(section: ScopeSection) -> SectionRow { + ( + section.name, + section.total, + section.returned, + section.collapsed, + section.collapsed_count, + section.truncated, + section.complete, + section.next_cursor, + section.cursor, + section.page_size, + ) +} + +fn section_in(row: SectionRow) -> ScopeSection { + let ( + name, + total, + returned, + collapsed, + collapsed_count, + truncated, + complete, + next_cursor, + cursor, + page_size, + ) = row; + ScopeSection { + name, + total, + returned, + collapsed, + collapsed_count, + truncated, + complete, + next_cursor, + cursor, + page_size, + } +} + +fn folder_row(folder: ScopeFolder) -> FolderRow { + ( + folder.path, + folder.name, + folder.descendant_count, + folder.child_count, + folder.used_count, + folder.kind_counts, + ) +} + +fn folder_in(row: FolderRow) -> ScopeFolder { + let (path, name, descendant_count, child_count, used_count, kind_counts) = row; + let mut unique: Vec<(String, i64)> = Vec::with_capacity(kind_counts.len()); + for (kind, count) in kind_counts { + match unique.iter_mut().find(|(existing, _)| *existing == kind) { + Some(entry) => entry.1 = count, + None => unique.push((kind, count)), + } + } + ScopeFolder { + path, + name, + descendant_count, + child_count, + used_count, + kind_counts: unique, + } +} + +pub(crate) fn explanation_out( + py: Python<'_>, + explanation: Explanation, +) -> PyResult> { + Ok(( + explanation + .declaration + .map(|report| declaration_out(py, report)) + .transpose()?, + explanation.complete, + )) +} + +fn explanation_in(row: ExplanationRow>) -> PyResult { + Ok(Explanation { + declaration: row.0.map(declaration_in).transpose()?, + complete: row.1, + }) +} + +pub(crate) fn move_out(py: Python<'_>, preview: MovePreview) -> PyResult> { + Ok(( + preview.resource, + preview.destination, + preview.new_ownership_root, + declarations_out(py, preview.retained)?, + declarations_out(py, preview.gained)?, + declarations_out(py, preview.lost)?, + preview.invalidated_usages, + declarations_out(py, preview.private_retained)?, + declarations_out(py, preview.relationship_retained)?, + preview.complete, + )) +} + +fn move_in(row: MoveRow>) -> PyResult { + let ( + resource, + destination, + new_ownership_root, + retained, + gained, + lost, + invalidated_usages, + private_retained, + relationship_retained, + complete, + ) = row; + Ok(MovePreview { + resource, + destination, + new_ownership_root, + retained: declarations_in(retained)?, + gained: declarations_in(gained)?, + lost: declarations_in(lost)?, + invalidated_usages, + private_retained: declarations_in(private_retained)?, + relationship_retained: declarations_in(relationship_retained)?, + complete, + }) +} + +fn resource_row(resource: ResourceReport) -> ResourceReportRow { + ( + resource.target, + resource.identity, + resource.path, + resource.prospective, + resource.directory, + resource.duplicate_count, + ) +} + +fn chain_row(entry: ScopeChainEntry) -> ChainRow { + (entry.kind, entry.path, entry.declaration_count) +} + +pub(crate) fn report_out(py: Python<'_>, report: ScopeReport) -> PyResult { + Ok(( + resource_row(report.resource), + report.scope_chain.into_iter().map(chain_row).collect(), + declarations_out(py, report.available)?, + declarations_out(py, report.used)?, + declarations_out(py, report.relationship_scope)?, + declarations_out(py, report.nearby_unavailable)?, + filters_row(&report.filters), + report.sections.into_iter().map(section_row).collect(), + report + .explanation + .map(|explanation| explanation_out(py, explanation)) + .transpose()?, + report + .move_preview + .map(|preview| move_out(py, preview)) + .transpose()?, + report.diagnostics.into_iter().map(diagnostic_out).collect(), + report.complete, + )) +} + +pub(crate) fn report_in( + row: ReportRow, DiagnosticInRow>, +) -> PyResult { + let ( + (target, identity, path, prospective, directory, duplicate_count), + chain, + available, + used, + relationship_scope, + nearby_unavailable, + filter_row, + sections, + explanation, + move_preview, + diagnostics, + complete, + ) = row; + Ok(ScopeReport { + resource: ResourceReport { + target, + identity, + path, + prospective, + directory, + duplicate_count, + }, + scope_chain: chain + .into_iter() + .map(|(kind, path, declaration_count)| ScopeChainEntry { + kind, + path, + declaration_count, + }) + .collect(), + available: declarations_in(available)?, + used: declarations_in(used)?, + relationship_scope: declarations_in(relationship_scope)?, + nearby_unavailable: declarations_in(nearby_unavailable)?, + filters: filters(filter_row), + sections: sections.into_iter().map(section_in).collect(), + explanation: explanation.map(explanation_in).transpose()?, + move_preview: move_preview.map(move_in).transpose()?, + diagnostics: diagnostics_in(diagnostics)?, + complete, + }) +} + +fn diagnostics_in(rows: Vec) -> PyResult> { + rows.into_iter() + .map(|row| Ok(ReportDiagnostic::Raised(diagnostic_facts(row)?))) + .collect() +} + +pub(crate) fn browse_out(result: BrowseResult) -> BrowseOutRow { + ( + result.folder, + result.folders.into_iter().map(folder_row).collect(), + result.diagnostics.into_iter().map(diagnostic_out).collect(), + result.complete, + ) +} + +pub(crate) fn browse_in(row: BrowseRow) -> PyResult { + let (folder, folders, diagnostics, complete) = row; + Ok(BrowseResult { + folder, + folders: folders.into_iter().map(folder_in).collect(), + diagnostics: diagnostics_in(diagnostics)?, + complete, + }) +} + +pub(crate) fn list_out(py: Python<'_>, result: ListResult) -> PyResult { + Ok(( + result.folder, + declarations_out(py, result.declarations)?, + section_row(result.section), + result.diagnostics.into_iter().map(diagnostic_out).collect(), + )) +} + +pub(crate) fn list_in(row: ListRow, DiagnosticInRow>) -> PyResult { + let (folder, declarations, section, diagnostics) = row; + Ok(ListResult { + folder, + declarations: declarations_in(declarations)?, + section: section_in(section), + diagnostics: diagnostics_in(diagnostics)?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs new file mode 100644 index 0000000000..30b5531a80 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -0,0 +1,346 @@ +//! `sqb scope` reports over a native scope lookup, their serialization and the metadata projection. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_scopes::scope_report::main::browse_scope_folders::browse_scope_folders; +use sqlbuild_scopes::scope_report::main::build_scope_report::build_scope_report; +use sqlbuild_scopes::scope_report::main::explain_scope_declaration::explain_scope_declaration; +use sqlbuild_scopes::scope_report::main::list_scope_declarations::list_scope_declarations; +use sqlbuild_scopes::scope_report::main::preview_scope_move::preview_scope_move; +use sqlbuild_scopes::scope_report::main::scope_metadata_projection::scope_metadata_projection; +use sqlbuild_scopes::scope_report::main::serialize_scope_result::serialize_scope_result; +use sqlbuild_scopes::scope_report::models::{ + ExplainRequest, FolderRequest, MoveRequest, ProjectionGrant, ProjectionIndex, + ProjectionStanding, ProjectionUsage, ReportFailure, ReportRequest, ReportSource, ReportValues, + ScopeResult, +}; + +use crate::bindings::_helpers::scopes::placement::{ + DeclarationRow, IdentityRow, QualifiedRow, ResourceRow, RootRow, declaration_identity, + declaration_record, ownership_root, qualified_identity, resource_identity, resource_record, +}; +use crate::bindings::_helpers::scopes::scope_lookup::{ + NativeScopeLookup, grant_kind, identity_message, path_message, +}; +use crate::bindings::_helpers::scopes::scope_report_rows::{ + BrowseOutRow, BrowseRow, CompletenessRow, DiagnosticInRow, ExplanationOutRow, FiltersRow, + ListOutRow, ListRow, MoveOutRow, ReportOutRow, ReportRow, TargetRow, ValuesRow, browse_in, + browse_out, completeness, declaration_target, declaration_values, diagnostic_facts, + explanation_out, filters, json_to_python, list_in, list_out, move_out, raised_out, report_in, + report_out, target, +}; + +type PyObject = Py; + +/// `(kind, message)` of the scope error Python raises: `path` or `identity`. +type FailureRow = Option<(&'static str, String)>; +/// Consumer, declaration, usage kind, what it came through and the enum member. +type UsageRow = ( + QualifiedRow, + QualifiedRow, + String, + Option, + Option, +); +/// Resource, declaration, through and grant kind. +type GrantRow = (IdentityRow, QualifiedRow, QualifiedRow, String); +/// Resource, declaration, reason and what it came through. +type StandingRow = (IdentityRow, QualifiedRow, String, Option); +/// Every `ScopeIndex` tuple, with declaration values parallel to the declarations. +type ProjectionRows = ( + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + CompletenessRow, +); +type ReportRequestRow = (Option, Option, bool, FiltersRow); +type ExplainRequestRow = ( + (Option, String), + Option, + Option, + bool, +); +type FolderRequestRow = (String, Option, bool, Option); + +fn failure_row(failure: &ReportFailure) -> PyResult { + match failure { + ReportFailure::Path(rejected) => Ok(Some(("path", path_message(rejected)))), + ReportFailure::Identity(failure, text) => { + Ok(Some(("identity", identity_message(failure, text)))) + } + ReportFailure::Serialization(message) => Err(PyValueError::new_err(message.clone())), + ReportFailure::IndexOutOfRange => { + Ok(Some(("index", "tuple index out of range".to_owned()))) + } + } +} + +/// One lookup's reports: the lookup and every declaration's values in input order. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct NativeScopeReports { + lookup: Py, + values: ReportValues, +} + +impl NativeScopeReports { + fn source<'a>(&'a self, lookup: &'a NativeScopeLookup) -> ReportSource<'a> { + ReportSource { + records: &lookup.records, + lookup: &lookup.lookup, + values: &self.values, + } + } +} + +#[pymethods] +impl NativeScopeReports { + /// `query_scope_report`. + fn report( + &self, + py: Python<'_>, + request: ReportRequestRow, + ) -> PyResult<(Option, FailureRow)> { + let (target_row, at, directory, filter_row) = request; + let request: ReportRequest = ReportRequest { + target: target_row.map(target).transpose()?, + at, + directory, + filters: filters(filter_row), + }; + let lookup = self.lookup.get(); + match build_scope_report(self.source(lookup), &request) { + Ok(report) => Ok((Some(report_out(py, report)?), None)), + Err(failure) => Ok((None, failure_row(&failure)?)), + } + } + + /// `explain_scope_declaration`. + fn explain( + &self, + py: Python<'_>, + request: ExplainRequestRow, + ) -> PyResult<(Option, FailureRow)> { + let (declaration, target_row, at, directory) = request; + let request: ExplainRequest = ExplainRequest { + declaration: declaration_target(declaration)?, + target: target_row.map(target).transpose()?, + at, + directory, + }; + let lookup = self.lookup.get(); + match explain_scope_declaration(self.source(lookup), &request) { + Ok((explanation, diagnostics)) => Ok(( + Some((explanation_out(py, explanation)?, raised_out(diagnostics))), + None, + )), + Err(failure) => Ok((None, failure_row(&failure)?)), + } + } + + /// `browse_scope_folders`. + fn browse(&self, request: FolderRequestRow) -> PyResult { + let request: FolderRequest = folder_request(request)?; + let lookup = self.lookup.get(); + Ok(browse_out(browse_scope_folders( + self.source(lookup), + &request, + ))) + } + + /// `list_scope_declarations`. + fn list( + &self, + py: Python<'_>, + request: FolderRequestRow, + ) -> PyResult<(Option, FailureRow)> { + let request: FolderRequest = folder_request(request)?; + let lookup = self.lookup.get(); + match list_scope_declarations(self.source(lookup), &request) { + Ok(result) => Ok((Some(list_out(py, result)?), None)), + Err(failure) => Ok((None, failure_row(&failure)?)), + } + } + + /// `preview_scope_move`. + fn move_preview( + &self, + py: Python<'_>, + request: (TargetRow, String), + ) -> PyResult<(Option, FailureRow)> { + let request: MoveRequest = MoveRequest { + resource: target(request.0)?, + destination: request.1, + }; + let lookup = self.lookup.get(); + match preview_scope_move(self.source(lookup), &request) { + Ok((preview, diagnostics)) => Ok(( + Some(( + preview.map(|preview| move_out(py, preview)).transpose()?, + raised_out(diagnostics), + )), + None, + )), + Err(failure) => Ok((None, failure_row(&failure)?)), + } + } +} + +fn folder_request(row: FolderRequestRow) -> PyResult { + let (folder, target_row, prospective, filter_row) = row; + Ok(FolderRequest { + folder, + target: target_row.map(target).transpose()?, + prospective, + filters: filter_row.map(filters), + }) +} + +/// The reports of `lookup`, with each canonical declaration's values and the completeness. +#[pyfunction] +fn build_native_scope_reports( + lookup: Py, + values: (Vec, CompletenessRow), +) -> PyResult { + let (rows, completeness_row) = values; + let native: &NativeScopeLookup = lookup.get(); + let canonical: &[usize] = &native.lookup.declarations; + if rows.len() != canonical.len() { + return Err(PyValueError::new_err( + "scope report values must match the lookup's declarations", + )); + } + let mut declarations = vec![Default::default(); canonical.len()]; + for (position, row) in canonical.iter().zip(rows) { + declarations[*position] = declaration_values(row)?; + } + Ok(NativeScopeReports { + lookup, + values: ReportValues { + declarations, + completeness: completeness(completeness_row), + }, + }) +} + +/// `serialize_scope_report` of a `report`, `browse` or `list` result row. +#[pyfunction] +fn serialize_scope_result_row(kind: &str, row: &Bound<'_, PyAny>) -> PyResult { + use pyo3::prelude::PyAnyMethods; + let result: ScopeResult = match kind { + "report" => ScopeResult::Report(Box::new(report_in( + row.extract::, DiagnosticInRow>>()?, + )?)), + "browse" => ScopeResult::Browse(browse_in(row.extract::>()?)?), + _ => ScopeResult::List(list_in( + row.extract::, DiagnosticInRow>>()?, + )?), + }; + serialize_scope_result(&result).map_err(|failure| match failure { + ReportFailure::Serialization(message) => PyValueError::new_err(message), + other => PyValueError::new_err(format!("{other:?}")), + }) +} + +fn projection_index(rows: ProjectionRows) -> PyResult { + let ( + roots, + resources, + declarations, + values, + usages, + grants, + visibility, + inaccessible, + diagnostics, + completeness_row, + ) = rows; + let standing = + |(resource, declaration, reason, through): StandingRow| -> PyResult { + Ok(ProjectionStanding { + resource: resource_identity(resource)?, + declaration: declaration_identity(declaration)?, + reason, + through: through.map(qualified_identity).transpose()?, + }) + }; + Ok(ProjectionIndex { + roots: roots + .into_iter() + .map(ownership_root) + .collect::>()?, + resources: resources + .into_iter() + .map(resource_record) + .collect::>()?, + declarations: declarations + .into_iter() + .map(declaration_record) + .collect::>()?, + values: ReportValues { + declarations: values + .into_iter() + .map(declaration_values) + .collect::>()?, + completeness: completeness(completeness_row), + }, + usages: usages + .into_iter() + .map(|(consumer, declaration, kind, through, enum_member)| { + Ok(ProjectionUsage { + consumer: qualified_identity(consumer)?, + declaration: declaration_identity(declaration)?, + kind, + through: through.map(qualified_identity).transpose()?, + enum_member, + }) + }) + .collect::>()?, + grants: grants + .into_iter() + .map(|(resource, declaration, through, kind)| { + Ok(ProjectionGrant { + resource: resource_identity(resource)?, + declaration: declaration_identity(declaration)?, + through: qualified_identity(through)?, + kind: grant_kind(&kind)?, + }) + }) + .collect::>()?, + visibility: visibility + .into_iter() + .map(standing) + .collect::>()?, + inaccessible: inaccessible + .into_iter() + .map(standing) + .collect::>()?, + diagnostics: diagnostics + .into_iter() + .map(diagnostic_facts) + .collect::>()?, + }) +} + +/// `scope_metadata_projection` of one index's rows, as the dictionary Python builds. +#[pyfunction] +fn scope_metadata_projection_rows(py: Python<'_>, rows: ProjectionRows) -> PyResult { + let index: ProjectionIndex = projection_index(rows)?; + let projection: JsonValue = scope_metadata_projection(&index); + json_to_python(py, projection, false) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(build_native_scope_reports, module)?)?; + module.add_function(wrap_pyfunction!(serialize_scope_result_row, module)?)?; + module.add_function(wrap_pyfunction!(scope_metadata_projection_rows, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 26edf624ae..b6f265a8da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -5,8 +5,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::semantic_checks::main::check_semantic_metadata::check_semantic_metadata; use sqlbuild_analysis::semantic_checks::models::{ - MetadataFinding, MetadataFunction, MetadataModel, MetadataRequest, MetadataSource, - MetadataSqlTest, ModelMetadataFindings, + AuditValue, MetadataAudit, MetadataFinding, MetadataFunction, MetadataModel, MetadataRequest, + MetadataSource, MetadataSqlTest, ModelMetadataFindings, }; use std::sync::Arc; @@ -17,7 +17,16 @@ use crate::bindings::types::CompilerDetach; /// `(key, names)` pairs in Python's order. type NamesInput = Vec<(String, Vec)>; -/// `(name, query, authored, checked, calls_functions, references, cursor_inputs, cursor, type)`. +/// `(column, definition, expression, to, field, values as (kind, text))`. +type AuditInput = ( + Option, + String, + Option, + String, + String, + Option>, +); +/// `(name, query, authored, checked, functions, references, cursor inputs, cursor, type, audits)`. type ModelInput = ( String, String, @@ -28,10 +37,12 @@ type ModelInput = ( NamesInput, Option, Option, + Vec, ); -/// `(dialect, return types, functions, shapes, models, file texts, sources, SQL tests)`. +/// `(dialect, quoted case, return types, functions, shapes, models, files, sources, SQL tests)`. type RequestInput = ( Option, + bool, Vec<(String, String)>, Vec<(String, Vec<(String, String)>)>, Vec<(String, Vec<(String, String)>)>, @@ -42,9 +53,11 @@ type RequestInput = ( ); /// `(code, message, line, column)`. type ErrorRow = (&'static str, String, i64, i64); -/// `([(function, reference findings)], source, SQL test findings, fallback types)`. +/// `(code, message, line, column, is error)`. +type AuditRow = (String, String, i64, i64, bool); +/// `([(function, audit, reference findings)], source, SQL test findings, fallback types)`. type OutcomeRow = ( - Vec<(Vec, Vec)>, + Vec<(Vec, Vec, Vec)>, Vec<(usize, ErrorRow)>, Vec<(usize, ErrorRow, i64)>, Vec, @@ -83,15 +96,38 @@ fn error_row(error: MetadataFinding) -> ErrorRow { (error.code, error.message, error.line, error.column) } -fn model_rows(errors: ModelMetadataFindings) -> (Vec, Vec) { +fn model_rows(errors: ModelMetadataFindings) -> (Vec, Vec, Vec) { ( errors.functions.into_iter().map(error_row).collect(), + errors + .audits + .into_iter() + .map(|finding| { + ( + finding.code, + finding.message, + finding.line, + finding.column, + finding.is_error, + ) + }) + .collect(), errors.references.into_iter().map(error_row).collect(), ) } fn metadata_request( - (dialect, return_types, functions, shapes, models, files, sources, sql_tests): RequestInput, + ( + dialect, + quoted_ignore_case, + return_types, + functions, + shapes, + models, + files, + sources, + sql_tests, + ): RequestInput, ) -> PyResult { let files: Vec> = files.into_iter().map(Arc::from).collect(); let file = |index: usize| -> PyResult> { @@ -102,6 +138,7 @@ fn metadata_request( }; Ok(MetadataRequest { dialect, + quoted_ignore_case, return_types, functions: functions .into_iter() @@ -142,6 +179,7 @@ fn metadata_model( cursor_inputs, cursor, cursor_type, + audits, ): ModelInput, ) -> MetadataModel { MetadataModel { @@ -154,6 +192,30 @@ fn metadata_model( cursor_inputs, cursor, cursor_type, + audits: audits.into_iter().map(metadata_audit).collect(), + } +} + +fn audit_value((kind, text): (String, String)) -> AuditValue { + match kind.as_str() { + "null" => AuditValue::Null, + "bool" => AuditValue::Bool(!text.is_empty()), + "number" => AuditValue::Number(text), + "text" => AuditValue::Text(text), + _ => AuditValue::Other, + } +} + +fn metadata_audit( + (column, definition, expression, target, field, values): AuditInput, +) -> MetadataAudit { + MetadataAudit { + column, + definition, + expression, + target, + field, + values: values.map(|values| values.into_iter().map(audit_value).collect()), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index b02c83a208..9ce91706e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -2,5 +2,6 @@ pub(crate) mod completion; pub(crate) mod failures; pub(crate) mod metadata; pub(crate) mod registration; +pub(crate) mod resource_sql; pub(crate) mod session_facts; pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs index b4c9ba8dae..10b484a3e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs @@ -2,10 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::semantic_checks::{completion, metadata, type_recovery}; +use crate::bindings::_helpers::semantic_checks::{ + completion, metadata, resource_sql, type_recovery, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { type_recovery::register(module)?; metadata::register(module)?; + resource_sql::register(module)?; completion::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs new file mode 100644 index 0000000000..3ef89e711a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -0,0 +1,205 @@ +//! Native audit, SQL test and SQL hook SQL checks against the closed model shapes. + +use std::sync::Arc; + +use pyo3::prelude::{ + Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::semantic_checks::main::resource_sql_diagnostics::resource_sql_diagnostics; +use sqlbuild_analysis::semantic_checks::models::{ + AuditSqlInput, HookModelInput, HookSqlInput, ResourceSqlDiagnostic, ResourceSqlFailure, + ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::models::ProjectCatalog; +use crate::bindings::types::CompilerDetach; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; + +type Shape = (String, Vec<(String, String)>); +/// `(name, definition, (target kind, target), column, built-in, path, contents, bodies, opts out, relations)`. +type AuditRow = ( + String, + String, + Option<(String, String)>, + Option, + bool, + String, + String, + Vec<(Option, Option)>, + bool, + Vec, +); +/// `(name, expected models, path, contents, compiled body, authored body, opts out)`. +type TestRow = (String, Vec, String, String, String, String, bool); +/// `(key, index, statement, hook path, definition, relations)`. +type HookRow = ( + String, + usize, + String, + Option, + Option, + Vec, +); +/// `(name, path, authored SQL, analysis disabled, hooks)`. +type ModelRow = (String, String, String, bool, Vec); +/// `(dialect, quoted ignore case, require analysis, shapes, audits, tests, models, hook files)`. +type RequestInput = ( + Option, + bool, + bool, + Vec, + Vec, + Vec, + Vec, + Vec<(String, String)>, +); +/// `(kind, name, is error, code, message, path, span, note, help)`. +type DiagnosticRow = ( + &'static str, + String, + bool, + String, + String, + String, + (i64, i64, Option, Option), + Option, + Option, +); + +fn audit(row: AuditRow) -> AuditSqlInput { + let ( + name, + definition_name, + attached_target, + attached_column, + built_in, + path, + contents, + bodies, + opts_out, + relations, + ) = row; + AuditSqlInput { + name, + definition_name, + attached_target, + attached_column, + built_in, + path, + contents: Arc::from(contents), + bodies, + opts_out, + relations, + } +} + +fn sql_test(row: TestRow) -> SqlTestSqlInput { + let (name, expected_model_names, path, contents, sql_body, authored_body, opts_out) = row; + SqlTestSqlInput { + name, + expected_model_names, + path, + contents: Arc::from(contents), + sql_body, + authored_body, + opts_out, + } +} + +fn model(row: ModelRow) -> HookModelInput { + let (name, path, authored_sql, analysis_disabled, hooks) = row; + HookModelInput { + name, + path, + authored_sql: Arc::from(authored_sql), + analysis_disabled, + hooks: hooks + .into_iter() + .map( + |(key, index, statement, hook_path, definition, relations)| HookSqlInput { + key, + index, + statement, + hook_path, + definition, + relations, + }, + ) + .collect(), + } +} + +fn request(input: RequestInput) -> ResourceSqlRequest { + let (dialect, quoted_ignore_case, require_sql_analysis, shapes, audits, tests, models, files) = + input; + ResourceSqlRequest { + dialect, + quoted_ignore_case, + require_sql_analysis, + shapes, + audits: audits.into_iter().map(audit).collect(), + sql_tests: tests.into_iter().map(sql_test).collect(), + models: models.into_iter().map(model).collect(), + hook_files: files + .into_iter() + .map(|(path, contents)| (path, Arc::from(contents))) + .collect(), + } +} + +fn row(diagnostic: ResourceSqlDiagnostic) -> DiagnosticRow { + let kind = match diagnostic.kind { + ResourceSqlKind::Audit => "audit", + ResourceSqlKind::SqlTest => "sql_test", + ResourceSqlKind::Model => "model", + }; + ( + kind, + diagnostic.resource_name, + diagnostic.is_error, + diagnostic.code, + diagnostic.message, + diagnostic.path, + diagnostic.span, + diagnostic.note, + diagnostic.help, + ) +} + +fn failure_error(py: Python<'_>, failure: ResourceSqlFailure) -> PyResult { + Ok(match failure { + ResourceSqlFailure::CursorIntrinsic(message) => PyErr::from_value( + py.import(COMPILE_ERRORS_MODULE)? + .getattr("CompileInputError")? + .call1((message,))?, + ), + ResourceSqlFailure::Normalization(message) | ResourceSqlFailure::Internal(message) => { + compiler_error(message) + } + }) +} + +/// Bind audit, SQL test and SQL hook bodies on `catalog` as Python's resource SQL checks do. +#[pyfunction] +fn resource_sql_diagnostic_rows( + py: Python<'_>, + catalog: PyRef<'_, ProjectCatalog>, + input: RequestInput, +) -> PyResult> { + let request = request(input); + let catalog = &catalog.inner; + let outcome = py + .compiler_detach(|| Ok(resource_sql_diagnostics(&request, catalog))) + .map_err(compiler_error)?; + match outcome { + Ok(diagnostics) => Ok(diagnostics.into_iter().map(row).collect()), + Err(failure) => Err(failure_error(py, failure)?), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(resource_sql_diagnostic_rows, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs index 381c3d0ab8..77dba08be1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs @@ -1,7 +1,68 @@ -//! Authored header values as the Python objects the model header parser builds. +//! Authored header values as the Python header objects: scalars, sets, constants and hooks. -use pyo3::prelude::{IntoPyObject, Py, PyAny, PyResult, Python}; +use pyo3::prelude::{IntoPyObject, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; +use sqlbuild_sqltext::compiler::models::HeaderWord; + +const SQL_VALUES_MODULE: &str = "sqlbuild.sql_values.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; + +fn word_to_python(py: Python<'_>, word: String) -> PyResult> { + match header_word_value(&word) { + HeaderWord::Bool(value) => Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()), + HeaderWord::Null => Ok(py.None()), + HeaderWord::Integer(digits) => match digits.parse::() { + Ok(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), + Err(_) => Ok(py + .import("builtins")? + .getattr("int")? + .call1((digits,))? + .unbind()), + }, + HeaderWord::Float(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), + HeaderWord::Text => Ok(word.into_pyobject(py)?.unbind().into_any()), + } +} + +fn instance( + py: Python<'_>, + (module, class): (&str, &str), + arguments: &pyo3::Bound<'_, PyDict>, +) -> PyResult> { + Ok(py + .import(module)? + .getattr(class)? + .call((), Some(arguments))? + .unbind()) +} + +fn values_tuple( + py: Python<'_>, + values: Vec, +) -> PyResult> { + let projected = values + .into_iter() + .map(|item| authored_value_to_python(py, item)) + .collect::>>()?; + Ok(PyTuple::new(py, projected)?.unbind().into_any()) +} + +fn constant_call( + py: Python<'_>, + values: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, +) -> PyResult> { + let arguments = values + .into_iter() + .map(|(key, value)| { + let value = authored_value_to_python(py, value)?; + PyTuple::new(py, [key.into_pyobject(py)?.unbind().into_any(), value]) + }) + .collect::>>()?; + let keywords = PyDict::new(py); + keywords.set_item("arguments", PyTuple::new(py, arguments)?)?; + instance(py, (SQL_VALUES_MODULE, "AuthoredSqlValueCall"), &keywords) +} fn authored_value_to_python( py: Python<'_>, @@ -14,9 +75,7 @@ fn authored_value_to_python( AuthoredValue::Boolean(value) => { Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()) } - AuthoredValue::BareWord(value) => { - marker_to_python(py, "word", value.into_pyobject(py)?.unbind().into_any()) - } + AuthoredValue::BareWord(value) => word_to_python(py, value), AuthoredValue::String(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), AuthoredValue::List(values) => { let projected = values @@ -26,26 +85,20 @@ fn authored_value_to_python( Ok(PyList::new(py, projected)?.unbind().into_any()) } AuthoredValue::Map(values) => map_to_python(py, values), - AuthoredValue::Set(values) => marker_to_python( - py, - "set", - authored_value_to_python(py, AuthoredValue::List(values))?, - ), - AuthoredValue::Tuple(values) => marker_to_python( - py, - "tuple", - authored_value_to_python(py, AuthoredValue::List(values))?, - ), - AuthoredValue::TypedConstant(values) => { - marker_to_python(py, "constant", map_to_python(py, values)?) + AuthoredValue::Set(values) => { + let keywords = PyDict::new(py); + keywords.set_item("values", values_tuple(py, values)?)?; + instance(py, (SQL_VALUES_MODULE, "AuthoredSqlSet"), &keywords) + } + AuthoredValue::Tuple(values) => values_tuple(py, values), + AuthoredValue::TypedConstant(values) => constant_call(py, values), + AuthoredValue::InlineSqlHook(statement) => { + let keywords = PyDict::new(py); + keywords.set_item("statement", statement)?; + instance(py, (DISCOVERY_MODELS_MODULE, "SqlHookEntry"), &keywords) } - AuthoredValue::InlineSqlHook(statement) => marker_to_python( - py, - "inline_sql", - statement.into_pyobject(py)?.unbind().into_any(), - ), - AuthoredValue::NamedSqlHook(name, kwargs) => hook_marker(py, "sql", name, kwargs), - AuthoredValue::PythonHook(name, kwargs) => hook_marker(py, "python", name, kwargs), + AuthoredValue::NamedSqlHook(name, kwargs) => hook(py, "NamedSqlHookEntry", name, kwargs), + AuthoredValue::PythonHook(name, kwargs) => hook(py, "PythonHookEntry", name, kwargs), } } @@ -60,28 +113,16 @@ pub(crate) fn map_to_python( Ok(result.unbind().into_any()) } -fn marker_to_python(py: Python<'_>, kind: &str, value: Py) -> PyResult> { - Ok( - PyTuple::new(py, [kind.into_pyobject(py)?.unbind().into_any(), value])? - .unbind() - .into_any(), - ) -} - -fn hook_marker( +fn hook( py: Python<'_>, - kind: &str, + class: &str, name: String, kwargs: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, ) -> PyResult> { - let payload = PyTuple::new( - py, - [ - name.into_pyobject(py)?.unbind().into_any(), - map_to_python(py, kwargs)?, - ], - )?; - marker_to_python(py, kind, payload.unbind().into_any()) + let keywords = PyDict::new(py); + keywords.set_item("name", name)?; + keywords.set_item("kwargs", map_to_python(py, kwargs)?)?; + instance(py, (DISCOVERY_MODELS_MODULE, class), &keywords) } pub(crate) fn optional_authored_value_to_python( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs index b01a3a7760..caf239bdf4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -4,6 +4,7 @@ use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references; use sqlbuild_sqltext::compiler::models::{ DeclarationReference, DeclarationReferenceKind, DeclarationReferenceScan, @@ -55,6 +56,13 @@ fn scanned_rows( Ok(scanned.into_iter().map(scan_row).collect()) } +/// One SQL string's references and stop code, scanned without walking past malformed ones. +pub(crate) fn declaration_scan(python: PythonText, sql: &str) -> ScanRow { + let scanned: Vec = + scan_declaration_references(python, &[sql.to_owned()], false); + scanned.into_iter().next().map(scan_row).unwrap_or_default() +} + fn scan_row(scan: DeclarationReferenceScan) -> ScanRow { ( scan.references.into_iter().map(reference_row).collect(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs index 60dabcfe4b..d760f5cd3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/lexical_syntax.rs @@ -4,7 +4,7 @@ use pyo3::FromPyObject; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; /// One adapter's `SqlLexicalSyntax`, read from a Python mapping. -#[derive(FromPyObject)] +#[derive(Clone, FromPyObject)] #[pyo3(from_item_all)] pub(crate) struct LexicalSyntaxInput { backslash_escape_quotes: Vec, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index 7a6740be79..edc0083cda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -51,9 +51,85 @@ fn tokenize_model_header(header: &str) -> PyResult> { }) } +type ColumnLocation = (String, usize, usize, usize, usize); + +fn location_rows( + locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, +) -> Vec { + locations + .into_iter() + .map(|(name, location)| { + ( + name, + location.line, + location.column, + location.end_line, + location.end_column, + ) + }) + .collect() +} + +/// Simple top-level SELECT output names and their authored locations. +#[pyfunction] +fn model_output_columns(contents: &str, implicit_aliases: bool) -> PyResult> { + compiler_guard(|| { + Ok(location_rows( + sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns( + contents, + implicit_aliases, + ), + )) + }) +} + +/// The authored locations of a header's column names, or the header's syntax error. +#[pyfunction] +fn model_header_column_locations( + contents: &str, + header: &str, + header_start: usize, +) -> PyResult<(Option>, Option)> { + compiler_guard(|| { + Ok( + match sqlbuild_sqltext::compiler::main::model_header_column_locations::model_header_column_locations( + contents, + header, + header_start, + ) { + Ok(locations) => (Some(location_rows(locations)), None), + Err(error) => (None, Some(error)), + }, + ) + }) +} + +/// A header syntax error's detail, help and file line (when `header_line` is known). +#[pyfunction] +#[pyo3(signature = (error, header, header_line))] +fn model_header_failure( + error: &str, + header: &str, + header_line: Option, +) -> (String, Option, Option) { + let nesting = sqlbuild_sqltext::compiler::main::model_header_errors::header_nesting_failure( + error, + header, + header_line.unwrap_or(1), + ); + match (nesting, header_line) { + (Some(nesting), Some(_)) => (nesting.message, Some(nesting.help), Some(nesting.line)), + (Some(nesting), None) => (error.to_owned(), Some(nesting.help), None), + (None, _) => (error.to_owned(), None, None), + } +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; module.add_function(wrap_pyfunction!(match_model_headers, module)?)?; module.add_function(wrap_pyfunction!(tokenize_model_header, module)?)?; + module.add_function(wrap_pyfunction!(model_output_columns, module)?)?; + module.add_function(wrap_pyfunction!(model_header_column_locations, module)?)?; + module.add_function(wrap_pyfunction!(model_header_failure, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index 3c0c3fa5e7..ecbe035c7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString, PyStringMethods}; use pyo3::{FromPyObject, PyErr, pyfunction, wrap_pyfunction}; use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; @@ -24,11 +25,11 @@ type InterpolationRow = ( /// Project variables, the environment, context values (or `None`) and the variable renderer. #[derive(FromPyObject)] -struct InterpolationSources<'py>( - Bound<'py, PyDict>, - Bound<'py, PyAny>, - Option>, - Bound<'py, PyAny>, +pub(crate) struct InterpolationSources<'py>( + pub(crate) Bound<'py, PyDict>, + pub(crate) Bound<'py, PyAny>, + pub(crate) Option>, + pub(crate) Bound<'py, PyAny>, ); struct PythonInterpolationHost<'py> { @@ -177,6 +178,30 @@ fn interpolate_sql_batch<'py>( Ok(rows) } +/// `(sql or None when unchanged, reads, error message or None)` for one SQL text. +pub(crate) type InterpolatedRow = (Option, Vec<(&'static str, String)>, Option); + +/// One SQL text interpolated: the SQL (`None` when unchanged), its reads, and its error. +pub(crate) fn interpolated( + python: PythonText, + sources: InterpolationSources<'_>, + sql: &str, + file_path: &str, +) -> PyResult { + let host = PythonInterpolationHost { + sources, + raised: RefCell::new(None), + }; + let row = match interpolate_sql(python, &host, sql, file_path) { + Ok(result) => (result.sql, read_rows(result.reads), None), + Err(failure) => (None, read_rows(failure.reads), Some(failure.message)), + }; + if let Some(error) = host.raised.borrow_mut().take() { + return Err(error); + } + Ok(row) +} + fn read_rows(reads: Vec) -> Vec<(&'static str, String)> { reads .into_iter() diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 5e1929b275..c5cbcbfce2 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -4,9 +4,9 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ - analysis, analysis_session, attachments, boundary, cache, contracts, discovery, lineage, - model_config, project_assembly, render, rules, scopes, semantic_checks, sql_test_glue, sqltext, - type_system, + analysis, analysis_session, attachments, boundary, cache, compile_outputs, compiled_project, + contracts, discovery, graph, lineage, model_config, project_assembly, refactor, render, rules, + scopes, semantic_checks, sql_test_glue, sqltext, type_system, }; use crate::bindings::models; @@ -15,7 +15,6 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { rules::evaluation::register(module)?; rules::lint::register(module)?; rules::skills::register(module)?; - module.add_class::()?; module.add( "NativeCompilerError", module.py().get_type::(), @@ -32,14 +31,20 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { scopes::registration::register(module)?; model_config::registration::register(module)?; render::registration::register(module)?; - cache::registration::register(module)?; - attachments::registration::register(module)?; + register_project_stages(module)?; register_analysis_stages(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) } +fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { + cache::registration::register(module)?; + attachments::registration::register(module)?; + refactor::registration::register(module)?; + Ok(()) +} + fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { boundary::python_runtime::record_python_text(module.py())?; type_system::registration::register(module)?; @@ -49,5 +54,8 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { lineage::registration::register(module)?; sql_test_glue::registration::register(module)?; project_assembly::registration::register(module)?; + compile_outputs::registration::register(module)?; + graph::registration::register(module)?; + compiled_project::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index 1d0ff9eb54..c87238fa69 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -3,17 +3,9 @@ use pyo3::{FromPyObject, pyclass}; use sqlbuild_analysis::semantic_validation::models as validation; use sqlbuild_analysis::semantic_validation::types::{Expansion, Relations}; -use sqlbuild_rules::models::EvaluateRequest; use std::collections::HashMap; use std::sync::Mutex; -/// A decoded rules request held natively so the caller can release its encoded payloads. -#[pyclass(module = "sqlbuild._native")] -#[derive(Debug)] -pub(crate) struct ParsedRulesRequest { - pub(crate) request: Mutex>, -} - #[derive(FromPyObject, Debug)] #[pyo3(from_item_all)] pub(crate) struct CatalogInput { diff --git a/crates/sqlbuild-refactor/Cargo.toml b/crates/sqlbuild-refactor/Cargo.toml new file mode 100644 index 0000000000..497a17b354 --- /dev/null +++ b/crates/sqlbuild-refactor/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "sqlbuild-refactor" +version.workspace = true +description = "Native `sqb rename` and `sqb mv` for SQLBuild: refactoring plans, text edits, staging and commit with Python's exact output." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true +sqlbuild-analysis.workspace = true +sqlbuild-config.workspace = true +sqlbuild-core.workspace = true +sqlbuild-model-config.workspace = true +sqlbuild-sqltext.workspace = true +tempfile.workspace = true diff --git a/crates/sqlbuild-refactor/src/lib.rs b/crates/sqlbuild-refactor/src/lib.rs new file mode 100644 index 0000000000..449aab4e2f --- /dev/null +++ b/crates/sqlbuild-refactor/src/lib.rs @@ -0,0 +1,5 @@ +//! Native `sqb rename` and `sqb mv`: plans, text edits, staging and commit over compiler facts. + +#![forbid(unsafe_code)] + +pub mod refactoring; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/mod.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/mod.rs new file mode 100644 index 0000000000..0ff2e56dc7 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod model_edits; +pub(crate) mod model_types; +pub(crate) mod source_edits; +pub(crate) mod yaml_lines; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/model_edits.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/model_edits.rs new file mode 100644 index 0000000000..d3934b4c3c --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/model_edits.rs @@ -0,0 +1,268 @@ +//! Contract adoption's MODEL header edit, as Python's `edit_model` plans it. + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_model_config::name_patterns::main::compile_name_pattern::compile_name_pattern; +use sqlbuild_sqltext::compiler::models::{HeaderColumnSpan, HeaderColumns}; + +use crate::refactoring::_helpers::contracts::model_types::{declared_type_span, render_model_type}; +use crate::refactoring::_helpers::edits::text_edits::{apply_text_edits, text_edit}; +use crate::refactoring::_helpers::scanning::chars::{find, slice}; +use crate::refactoring::constants::{ + CONTRACT_DEFAULT_INDENT, CONTRACT_HEADER_WHITESPACE, CONTRACT_OVERLAP_MESSAGE, + EXTERNAL_SCHEMA_CONFLICT, INVALID_NAME_PATTERN_PREFIX, SHARED_SCHEMA_CONFLICT, +}; +use crate::refactoring::errors::{RefactorError, RefactorErrorKind}; +use crate::refactoring::models::{ + ContractEdit, ContractModelColumn, ContractModelRequest, EditKind, TextEdit, +}; +use crate::refactoring::types::PhysicalColumn; + +/// A Python dict built by assignment: each key keeps its first position and its last value. +pub(crate) fn python_dict<'a, T>( + items: impl Iterator, +) -> Vec<(String, &'a T)> { + let mut entries: Vec<(String, &'a T)> = Vec::new(); + for (key, value) in items { + match entries.iter_mut().find(|(existing, _)| *existing == key) { + Some(entry) => entry.1 = value, + None => entries.push((key, value)), + } + } + entries +} + +/// The conflict `edit_model` reports before reading the header, if any. +pub(crate) fn ownership_conflict(request: &ContractModelRequest) -> Option { + if request.shared_schema { + return Some(ContractEdit::Conflict(SHARED_SCHEMA_CONFLICT.to_owned())); + } + request + .columns + .iter() + .any(|column| { + column + .location_path + .as_ref() + .is_some_and(|path| *path != request.relative_path) + }) + .then(|| ContractEdit::Conflict(EXTERNAL_SCHEMA_CONFLICT.to_owned())) +} + +/// Whether any family accepts `name`; every pattern is compiled, as Python's tuple filter does. +fn family_member(patterns: &[Option], name: &str) -> Result { + let mut member = false; + for pattern in patterns { + member |= match pattern { + None => true, + Some(pattern) => compile_name_pattern(pattern) + .map_err(|reason| { + RefactorError::value(format!("{INVALID_NAME_PATTERN_PREFIX}{reason}")) + })? + .matches(name), + }; + } + Ok(member) +} + +/// Physical columns without the runtime members of declared dynamic families. +pub(crate) fn fixed_physical_columns( + request: &ContractModelRequest, +) -> Result, RefactorError> { + if request.dynamic_patterns.is_empty() { + return Ok(request.physical_columns.clone()); + } + let python = active_python_text(); + let fixed: Vec = request + .columns + .iter() + .map(|column| python_casefold(python, &column.name)) + .collect(); + let mut kept: Vec = Vec::new(); + for column in &request.physical_columns { + if fixed.contains(&python_casefold(python, &column.0)) + || !family_member(&request.dynamic_patterns, &column.0)? + { + kept.push(column.clone()); + } + } + Ok(kept) +} + +/// Python's `_render_model_columns` inserted after the header body: `\n columns (...)`. +pub(crate) fn columns_block(columns: &[PhysicalColumn]) -> String { + let body: String = columns + .iter() + .map(|(name, column_type)| { + format!("\n {name} (type {})", render_model_type(column_type)) + }) + .collect(); + format!("\n columns ({body}\n )") +} + +/// Python's `_column_indentation`: the indent of the line after the columns body opens. +fn column_indentation(text: &[char], body_start: usize) -> String { + let Some(next_line) = find(text, "\n", body_start) else { + return CONTRACT_DEFAULT_INDENT.to_owned(); + }; + let indentation: String = text[next_line + 1..] + .iter() + .take_while(|character| CONTRACT_HEADER_WHITESPACE.contains(character)) + .collect(); + if indentation.is_empty() { + CONTRACT_DEFAULT_INDENT.to_owned() + } else { + indentation + } +} + +fn edit(text: &[char], span: (usize, usize), replacement: String) -> TextEdit { + text_edit(text, span, replacement, EditKind::Column) +} + +/// The edit one declared column needs, `Ok(None)` for none, or the conflict that stops all edits. +fn declared_column_edit( + text: &[char], + column: &ContractModelColumn, + (span, physical_type): (&HeaderColumnSpan, Option<&str>), + overwrite: bool, +) -> Result, String> { + let Some(physical_type) = physical_type else { + return Ok(overwrite.then(|| edit(text, (span.entry_start, span.entry_end), String::new()))); + }; + let python = active_python_text(); + match &column.declared_type { + None => { + let metadata = slice(text, span.metadata_start, span.metadata_end); + let separator = if python_strip(&metadata).is_empty() { + "" + } else { + ", " + }; + Ok(Some(edit( + text, + (span.metadata_start, span.metadata_start), + format!("type {}{separator}", render_model_type(physical_type)), + ))) + } + Some(declared) + if overwrite + && python_casefold(python, declared) != python_casefold(python, physical_type) => + { + declared_type_span(text, (span.metadata_start, span.metadata_end), declared) + .map(|replacement| Some(edit(text, replacement, render_model_type(physical_type)))) + .ok_or_else(|| { + format!( + "column '{}' type span could not be edited safely", + column.name + ) + }) + } + Some(_) => Ok(None), + } +} + +/// The edits of every declared column, in Python's dict order, or the first conflict. +fn declared_edits( + text: &[char], + request: &ContractModelRequest, + columns: &HeaderColumns, + physical: &[PhysicalColumn], +) -> Result, String> { + let python = active_python_text(); + let declared = python_dict( + request + .columns + .iter() + .map(|column| (python_casefold(python, &column.name), column)), + ); + let physical_by_name = python_dict( + physical + .iter() + .map(|column| (python_casefold(python, &column.0), column)), + ); + let mut edits: Vec = Vec::new(); + for (name, column) in &declared { + let Some((_, span)) = columns + .entries + .iter() + .rev() + .find(|(entry, _)| *entry == column.name) + else { + return Err(format!( + "column '{}' has no unambiguous authored span", + column.name + )); + }; + let physical_type = physical_by_name + .iter() + .find(|(key, _)| key == name) + .map(|(_, (_, column_type))| column_type.as_str()); + edits.extend(declared_column_edit( + text, + column, + (span, physical_type), + request.overwrite, + )?); + } + Ok(edits) +} + +/// The physical columns no declared column names, rendered as new `columns` entries. +fn additions( + text: &[char], + columns: &HeaderColumns, + request: &ContractModelRequest, + physical: &[PhysicalColumn], +) -> String { + let python = active_python_text(); + let declared: Vec = request + .columns + .iter() + .map(|column| python_casefold(python, &column.name)) + .collect(); + let indentation = column_indentation(text, columns.body_start); + let mut rendered = String::new(); + for (name, column_type) in physical { + if !declared.contains(&python_casefold(python, name)) { + rendered.push_str(&format!( + "\n{indentation}{name} (type {})", + render_model_type(column_type) + )); + } + } + rendered +} + +/// Python's `edit_model` once the header declares `columns (...)`. +pub(crate) fn columns_edit( + text: &[char], + request: &ContractModelRequest, + columns: &HeaderColumns, + physical: &[PhysicalColumn], +) -> Result { + let mut edits = match declared_edits(text, request, columns, physical) { + Ok(edits) => edits, + Err(conflict) => return Ok(ContractEdit::Conflict(conflict)), + }; + let added = additions(text, columns, request, physical); + if !added.is_empty() { + edits.push(edit( + text, + (columns.body_end, columns.body_end), + format!("{added}\n "), + )); + } + edits.sort_by(|left, right| { + (right.start, right.end, &right.replacement).cmp(&(left.start, left.end, &left.replacement)) + }); + match apply_text_edits(text, &edits) { + Ok(edited) => Ok(ContractEdit::Edited(edited.into_iter().collect())), + Err(error) if error.kind == RefactorErrorKind::Edit => Err(RefactorError { + message: CONTRACT_OVERLAP_MESSAGE.to_owned(), + ..error + }), + Err(error) => Err(error), + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/model_types.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/model_types.rs new file mode 100644 index 0000000000..650b66666f --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/model_types.rs @@ -0,0 +1,227 @@ +//! MODEL header column types: rendering, matching and the authored span of a declared type. + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::python_casefold::python_casefold; + +use crate::refactoring::_helpers::scanning::chars::slice; +use crate::refactoring::constants::{ + CONTRACT_CLOSERS, CONTRACT_HEADER_WORD_EXTRA, CONTRACT_MODEL_HEADER_ESCAPE, CONTRACT_OPENERS, + CONTRACT_QUOTES, CONTRACT_TYPE_KEY, +}; + +fn is_type_name_start(character: char) -> bool { + character.is_ascii_alphabetic() || character == '_' +} + +fn is_type_name_character(character: char) -> bool { + character.is_ascii_alphanumeric() || matches!(character, '_' | '$') +} + +fn is_type_argument_character(character: char) -> bool { + is_type_name_character(character) || matches!(character, '+' | '-') +} + +fn skip_space(text: &[char], index: usize) -> usize { + index + + text[index..] + .iter() + .take_while(|character| is_python_space(**character)) + .count() +} + +fn skip_argument(text: &[char], index: usize) -> Option { + let length = text[index..] + .iter() + .take_while(|character| is_type_argument_character(**character)) + .count(); + (length > 0).then_some(index + length) +} + +/// Python's `_UNQUOTED_MODEL_TYPE_PATTERN.fullmatch`: a name with optional `(arg, ...)`. +pub(crate) fn is_unquoted_type(value: &str) -> bool { + let text: Vec = value.chars().collect(); + if !text + .first() + .is_some_and(|character| is_type_name_start(*character)) + { + return false; + } + let mut index = 1 + text[1..] + .iter() + .take_while(|character| is_type_name_character(**character)) + .count(); + if index == text.len() { + return true; + } + index = skip_space(&text, index); + if text.get(index) != Some(&'(') { + return false; + } + index = skip_space(&text, index + 1); + let Some(mut index) = skip_argument(&text, index) else { + return false; + }; + loop { + let after_space = skip_space(&text, index); + if text.get(after_space) != Some(&',') { + index = after_space; + break; + } + let Some(next) = skip_argument(&text, skip_space(&text, after_space + 1)) else { + return false; + }; + index = next; + } + text.get(index) == Some(&')') && index + 1 == text.len() +} + +/// Python's `_render_model_type`: bare when unquoted-safe, else a double-quoted string. +pub(crate) fn render_model_type(value: &str) -> String { + if is_unquoted_type(value) { + return value.to_owned(); + } + let escaped = value.replace('\\', "\\\\").replace('"', "\\\""); + format!("\"{escaped}\"") +} + +fn without_space(value: &str) -> String { + value + .chars() + .filter(|character| !is_python_space(*character)) + .collect() +} + +/// Python's `_model_types_match`. +fn model_types_match(authored: &str, parsed: &str) -> bool { + let python = active_python_text(); + if python_casefold(python, authored) == python_casefold(python, parsed) { + return true; + } + if !(is_unquoted_type(authored) && is_unquoted_type(parsed)) { + return false; + } + python_casefold(python, &without_space(authored)) + == python_casefold(python, &without_space(parsed)) +} + +fn is_header_word_character(character: char) -> bool { + is_python_alnum(active_python_text(), character) + || CONTRACT_HEADER_WORD_EXTRA.contains(&character) +} + +fn quoted_value_end(text: &[char], start: usize, end: usize) -> usize { + let quote = text[start]; + let mut index = start + 1; + while index < end { + if text[index] == CONTRACT_MODEL_HEADER_ESCAPE { + index += 2; + continue; + } + if text[index] == quote { + return index + 1; + } + index += 1; + } + end +} + +fn unquoted_type_value_end(text: &[char], start: usize, end: usize) -> usize { + let mut index = start; + let mut depth = 0_usize; + let mut saw_open = false; + while index < end { + let character = text[index]; + if character == '(' { + depth += 1; + saw_open = true; + } else if character == ')' { + if depth == 0 { + break; + } + depth -= 1; + } else if depth == 0 && is_python_space(character) { + let next = index + + text[index..end] + .iter() + .take_while(|character| is_python_space(**character)) + .count(); + if !saw_open && text.get(next) == Some(&'(') { + index = next; + continue; + } + break; + } else if depth == 0 && character == ',' { + break; + } + index += 1; + } + index +} + +/// Python's `_declared_type_span`: the authored top-level `type` value matching `declared`. +pub(crate) fn declared_type_span( + text: &[char], + metadata: (usize, usize), + declared: &str, +) -> Option<(usize, usize)> { + let (metadata_start, metadata_end) = metadata; + let python = active_python_text(); + let key_length = CONTRACT_TYPE_KEY.chars().count(); + let mut index = metadata_start; + let mut depth = 0_usize; + while index < metadata_end { + let character = text[index]; + if CONTRACT_QUOTES.contains(&character) { + index = quoted_value_end(text, index, metadata_end); + continue; + } + if CONTRACT_OPENERS.contains(&character) { + depth += 1; + index += 1; + continue; + } + if CONTRACT_CLOSERS.contains(&character) { + depth = depth.saturating_sub(1); + index += 1; + continue; + } + if depth > 0 + || python_casefold(python, &slice(text, index, index + key_length)) != CONTRACT_TYPE_KEY + { + index += 1; + continue; + } + let key_end = index + key_length; + if (index > metadata_start && is_header_word_character(text[index - 1])) + || (key_end < metadata_end && is_header_word_character(text[key_end])) + { + index = key_end; + continue; + } + let mut value_start = key_end; + while value_start < metadata_end && is_python_space(text[value_start]) { + value_start += 1; + } + let (value_end, authored) = match text.get(value_start) { + Some(quote) if CONTRACT_QUOTES.contains(quote) => { + let value_end = quoted_value_end(text, value_start, metadata_end); + let inner = slice(text, value_start + 1, value_end.saturating_sub(1)); + let authored = inner + .replace(&format!("\\{quote}"), "e.to_string()) + .replace("\\\\", "\\"); + (value_end, authored) + } + _ => { + let value_end = unquoted_type_value_end(text, value_start, metadata_end); + (value_end, slice(text, value_start, value_end)) + } + }; + if model_types_match(&authored, declared) { + return Some((value_start, value_end)); + } + index = value_end; + } + None +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/source_edits.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/source_edits.rs new file mode 100644 index 0000000000..f0f903681f --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/source_edits.rs @@ -0,0 +1,188 @@ +//! Contract adoption's YAML source declaration edit, as Python's `edit_source` plans it. + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_core::text::main::python_splitlines::python_splitlines; + +use crate::refactoring::_helpers::contracts::model_edits::python_dict; +use crate::refactoring::_helpers::contracts::yaml_lines::{ + column_lines, comment, indent, is_content, key, list_name, sequence_indent, yaml_string, +}; +use crate::refactoring::constants::{ + CONTRACT_GENERATED_MARKER, CONTRACT_GENERATED_MARKER_LINES, CONTRACT_TYPE_KEY, + CONTRACT_YAML_COLUMNS_KEY, +}; +use crate::refactoring::models::ContractSourceRequest; +use crate::refactoring::types::LineEdit; + +/// Whether the first lines of the discovered file mark it as generated. +pub(crate) fn is_generated(contents: &str) -> bool { + let head: Vec<&str> = python_splitlines(contents) + .into_iter() + .take(CONTRACT_GENERATED_MARKER_LINES) + .collect(); + python_casefold(active_python_text(), &head.join("\n")).contains(CONTRACT_GENERATED_MARKER) +} + +/// The `columns:` key directly under the source block `(start, end, indent)`. +pub(crate) fn columns_key( + lines: &[&str], + (start, end, source_indent): (usize, usize, usize), +) -> Option { + (start + 1..end).find(|index| { + key(lines[*index]) == Some(CONTRACT_YAML_COLUMNS_KEY) + && indent(lines[*index]) == source_indent + 2 + }) +} + +/// A new `columns:` key holding every physical column, inserted at the source block's end. +pub(crate) fn columns_insertion( + request: &ContractSourceRequest, + source_indent: usize, +) -> Vec { + let mut insertion = vec![format!("{}columns:\n", " ".repeat(source_indent + 2))]; + for (name, column_type) in &request.physical_columns { + insertion.extend(column_lines(name, column_type, source_indent + 4)); + } + insertion +} + +/// The end of the `columns:` block: the first content line that leaves it. +fn columns_end( + lines: &[&str], + columns_index: usize, + column_indent: usize, + source_end: usize, +) -> usize { + let columns_indent = indent(lines[columns_index]); + (columns_index + 1..source_end) + .find(|index| { + let line = lines[*index]; + let line_indent = indent(line); + is_content(line) + && (line_indent < columns_indent + || (line_indent == columns_indent + && !(column_indent == columns_indent && list_name(line).is_some()))) + }) + .unwrap_or(source_end) +} + +/// Each casefolded column name's `(start, end)` line block, in Python's dict order. +fn column_blocks( + lines: &[&str], + (start, end): (usize, usize), + column_indent: usize, +) -> Vec<(String, (usize, usize))> { + let python = active_python_text(); + let mut found: Vec<(String, (usize, usize))> = Vec::new(); + let mut index = start; + while index < end { + match list_name(lines[index]) { + Some(name) if indent(lines[index]) == column_indent => { + let block_end = (index + 1..end) + .find(|line| { + indent(lines[*line]) <= column_indent && list_name(lines[*line]).is_some() + }) + .unwrap_or(end); + found.push((python_casefold(python, name), (index, block_end))); + index = block_end; + } + _ => index += 1, + } + } + python_dict(found.iter().map(|(name, block)| (name.clone(), block))) + .into_iter() + .map(|(name, block)| (name, *block)) + .collect() +} + +/// The type-line edit of one declared column whose physical column exists. +fn type_edit( + lines: &[&str], + block: (usize, usize), + (declared_type, physical_type): (Option<&str>, &str), + (column_indent, overwrite): (usize, bool), +) -> Option { + let python = active_python_text(); + let typed = format!( + "{}type: {}", + " ".repeat(column_indent + 2), + yaml_string(physical_type) + ); + match (block.0 + 1..block.1).find(|line| key(lines[*line]) == Some(CONTRACT_TYPE_KEY)) { + None => Some((block.0 + 1, block.0 + 1, vec![format!("{typed}\n")])), + Some(type_line) + if overwrite + && declared_type.is_some_and(|declared| { + python_casefold(python, declared) != python_casefold(python, physical_type) + }) => + { + let line = lines[type_line]; + let newline = if line.ends_with('\n') { "\n" } else { "" }; + Some(( + type_line, + type_line + 1, + vec![format!("{typed}{}{newline}", comment(line))], + )) + } + Some(_) => None, + } +} + +/// The line edits of the `columns:` key at `columns_index`, or the conflict that stops them. +pub(crate) fn column_edits( + request: &ContractSourceRequest, + lines: &[&str], + (columns_index, source_end): (usize, usize), +) -> Result, String> { + let python = active_python_text(); + let column_indent = sequence_indent( + lines, + (columns_index + 1, source_end), + indent(lines[columns_index]), + ) + .unwrap_or(indent(lines[columns_index]) + 2); + let end = columns_end(lines, columns_index, column_indent, source_end); + let blocks = column_blocks(lines, (columns_index + 1, end), column_indent); + let declared = python_dict( + request + .columns + .iter() + .map(|column| (python_casefold(python, &column.0), column)), + ); + let physical = python_dict( + request + .physical_columns + .iter() + .map(|column| (python_casefold(python, &column.0), column)), + ); + let mut line_edits: Vec = Vec::new(); + for (name, (declared_name, declared_type)) in &declared { + let Some((_, block)) = blocks.iter().find(|(key, _)| key == name) else { + return Err(format!( + "source column '{declared_name}' has no unambiguous YAML block" + )); + }; + match physical.iter().find(|(key, _)| key == name) { + None if request.overwrite => line_edits.push((block.0, block.1, Vec::new())), + None => {} + Some((_, (_, physical_type))) => line_edits.extend(type_edit( + lines, + *block, + (declared_type.as_deref(), physical_type), + (column_indent, request.overwrite), + )), + } + } + let mut insertion: Vec = Vec::new(); + for (name, column_type) in &request.physical_columns { + let key = python_casefold(python, name); + if !declared.iter().any(|(declared, _)| *declared == key) { + insertion.extend(column_lines(name, column_type, column_indent)); + } + } + if !insertion.is_empty() { + line_edits.push((end, end, insertion)); + } + Ok(line_edits) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/yaml_lines.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/yaml_lines.rs new file mode 100644 index 0000000000..ab2121ac12 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/contracts/yaml_lines.rs @@ -0,0 +1,134 @@ +//! Line-based YAML source declaration reads, as contract adoption's `edit_source` makes them. + +use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::refactoring::constants::{ + CONTRACT_QUOTES, CONTRACT_YAML_COMMENT, CONTRACT_YAML_LIST_NAME, +}; + +/// Python's `_indent`: the leading spaces of a line, in code points. +pub(crate) fn indent(line: &str) -> usize { + line.chars() + .take_while(|character| *character == ' ') + .count() +} + +fn unquote(value: &str) -> &str { + let mut characters = value.chars(); + match (characters.next(), characters.next_back()) { + (Some(first), Some(last)) if first == last && CONTRACT_QUOTES.contains(&first) => { + &value[first.len_utf8()..value.len() - last.len_utf8()] + } + _ => value, + } +} + +/// Python's `_yaml_list_name`: the unquoted name of a `- name:` line. +pub(crate) fn list_name(line: &str) -> Option<&str> { + python_strip(line) + .strip_prefix(CONTRACT_YAML_LIST_NAME) + .map(|rest| unquote(python_strip(rest))) +} + +/// Python's `_yaml_key`: the key of a `key: value` line. +pub(crate) fn key(line: &str) -> Option<&str> { + let stripped = python_strip(line); + if stripped.is_empty() || stripped.starts_with(['#', '-']) { + return None; + } + stripped.split_once(':').map(|(key, _)| python_strip(key)) +} + +/// Python's `_yaml_named_block`: the one shallowest `- name: ` block. +pub(crate) fn named_block(lines: &[&str], name: &str) -> Option<(usize, usize, usize)> { + let matches: Vec<(usize, usize)> = lines + .iter() + .enumerate() + .filter(|(_, line)| list_name(line) == Some(name)) + .map(|(index, line)| (index, indent(line))) + .collect(); + let minimum = matches.iter().map(|(_, indent)| *indent).min()?; + let candidates: Vec = matches + .iter() + .filter(|(_, indent)| *indent == minimum) + .map(|(index, _)| *index) + .collect(); + let [start] = candidates[..] else { + return None; + }; + let end = (start + 1..lines.len()) + .find(|index| indent(lines[*index]) <= minimum && list_name(lines[*index]).is_some()) + .unwrap_or(lines.len()); + Some((start, end, minimum)) +} + +fn is_blank_or_comment(line: &str) -> bool { + let stripped = python_strip(line); + stripped.is_empty() || stripped.starts_with('#') +} + +/// Python's `_yaml_sequence_indent`: the indent of the first `- name:` item under a key. +pub(crate) fn sequence_indent( + lines: &[&str], + range: (usize, usize), + key_indent: usize, +) -> Option { + for line in &lines[range.0..range.1] { + if is_blank_or_comment(line) { + continue; + } + let line_indent = indent(line); + if list_name(line).is_some() && line_indent >= key_indent { + return Some(line_indent); + } + if line_indent <= key_indent { + return None; + } + } + None +} + +/// Whether a line is neither blank nor a comment, as the columns-block scan reads it. +pub(crate) fn is_content(line: &str) -> bool { + !is_blank_or_comment(line) +} + +/// Python's `_yaml_comment`: a trailing ` #` comment without its line breaks. +pub(crate) fn comment(line: &str) -> &str { + line.find(CONTRACT_YAML_COMMENT) + .map_or("", |marker| line[marker..].trim_end_matches('\n')) +} + +/// Python's `_yaml_string`: a single-quoted YAML scalar. +pub(crate) fn yaml_string(value: &str) -> String { + format!("'{}'", value.replace('\'', "''")) +} + +/// Python's `_yaml_column_lines`. +pub(crate) fn column_lines(name: &str, column_type: &str, indent: usize) -> Vec { + vec![ + format!("{}- name: {name}\n", " ".repeat(indent)), + format!( + "{}type: {}\n", + " ".repeat(indent + 2), + yaml_string(column_type) + ), + ] +} + +/// Python's `str.splitlines(keepends=True)`: the lines of `text` with their breaks. +pub(crate) fn lines_with_breaks(text: &str) -> Vec { + let base = text.as_ptr() as usize; + let starts: Vec = python_splitlines(text) + .iter() + .map(|line| line.as_ptr() as usize - base) + .collect(); + starts + .iter() + .enumerate() + .map(|(index, start)| { + text[*start..starts.get(index + 1).copied().unwrap_or(text.len())].to_owned() + }) + .collect() +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/header_edits.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/header_edits.rs new file mode 100644 index 0000000000..16fef62bf6 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/header_edits.rs @@ -0,0 +1,626 @@ +//! Token-exact MODEL and SCHEMA header edits, as `header_edits.py` and `schema_edits.py` make. + +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::is_python_word::is_python_word; +use sqlbuild_core::text::models::PythonText; +use sqlbuild_sqltext::compiler::main::model_header_matching::match_batch; +use sqlbuild_sqltext::compiler::main::model_header_tokenizing::tokenize_one; +use sqlbuild_sqltext::compiler::main::statement_header_matching::match_statement_header; + +use crate::refactoring::_helpers::edits::text_edits::{text_edit, whole_word_offsets}; +use crate::refactoring::_helpers::scanning::chars::{find, rfind, slice, starts_with}; +use crate::refactoring::_helpers::scanning::sql_sites::embedded_ref_spans; +use crate::refactoring::constants::{ + COLUMN_VALUED_CONFIG_KEYS, COLUMNS_KEY, CURSOR_INPUTS_KEY, HEADER_CLOSERS, + HEADER_DESCRIPTION_KEY, HEADER_INDENT, HEADER_KEY_AND_VALUE_TOKENS, HEADER_OPEN_PAREN, + HEADER_OPENERS, HEADER_SEPARATOR, MIGRATE_FROM_KEY, PARENTHESIZED_EMPTY_TOKENS, REF_FUNCTION, + RELATIONSHIPS_AUDIT, RELATIONSHIPS_FIELD_KEY, RELATIONSHIPS_TO_KEY, SCHEMA_KEYWORD, +}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{EditKind, TextEdit}; + +const END_TOKEN: u8 = 0; +const WORD_TOKEN: u8 = 1; +const STRING_TOKEN: u8 = 2; +const SYMBOL_TOKEN: u8 = 3; + +/// MODEL header token kinds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum HeaderTokenKind { + Word, + String, + Symbol, +} + +/// One header token in file offsets, with its nesting depth. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct HeaderToken { + pub(crate) kind: HeaderTokenKind, + pub(crate) value: String, + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) depth: i64, +} + +type NestedEntry<'a> = (&'a HeaderToken, Vec<&'a HeaderToken>); + +/// The relationships audit target and field value tokens. +struct RelationshipTokens<'a> { + target: Option<&'a HeaderToken>, + field: Option<&'a HeaderToken>, + called: bool, +} + +fn is_value_kind(kind: HeaderTokenKind) -> bool { + matches!(kind, HeaderTokenKind::Word | HeaderTokenKind::String) +} + +/// The code-point span of the MODEL header body, as `get_model_header_spans(...).body`. +pub(crate) fn model_header_body_span(contents: &str) -> Option<(usize, usize)> { + match_batch(&[contents.to_owned()]) + .into_iter() + .next() + .flatten() + .map(|(start, end, _)| (start, end)) +} + +/// Tokenize the MODEL header body in file offsets, or `None` without a header. +pub(crate) fn header_tokens( + contents: &str, + text: &[char], +) -> Result>, RefactorError> { + match model_header_body_span(contents) { + Some(span) => span_tokens(text, span).map(Some), + None => Ok(None), + } +} + +/// Tokenize one declaration header body in file offsets. +pub(crate) fn span_tokens( + text: &[char], + span: (usize, usize), +) -> Result, RefactorError> { + let body = slice(text, span.0, span.1); + let raw = tokenize_one(&body).map_err(RefactorError::value)?; + let mut tokens: Vec = Vec::new(); + let mut depth: i64 = 0; + for (raw_kind, value, position) in raw { + let kind = match raw_kind { + END_TOKEN => continue, + WORD_TOKEN => HeaderTokenKind::Word, + STRING_TOKEN => HeaderTokenKind::String, + SYMBOL_TOKEN => HeaderTokenKind::Symbol, + _ => continue, + }; + let start = span.0 + position; + let end = if kind == HeaderTokenKind::String { + quoted_value_end(text, start) + } else { + start + value.chars().count() + }; + if kind == HeaderTokenKind::Symbol && HEADER_CLOSERS.contains(&value.as_str()) { + depth -= 1; + } + let opens = kind == HeaderTokenKind::Symbol && HEADER_OPENERS.contains(&value.as_str()); + tokens.push(HeaderToken { + kind, + value, + start, + end, + depth, + }); + if opens { + depth += 1; + } + } + Ok(tokens) +} + +/// The offset just past the quoted value that opens at `start` (`quoted_value_end`). +fn quoted_value_end(text: &[char], start: usize) -> usize { + let Some("e) = text.get(start) else { + return text.len(); + }; + let mut index = start + 1; + while index < text.len() { + if text[index] == '\\' && index + 1 < text.len() { + index += 2; + continue; + } + if text[index] == quote { + return index + 1; + } + index += 1; + } + text.len() +} + +fn is_separator(token: &HeaderToken) -> bool { + token.kind == HeaderTokenKind::Symbol && token.value == HEADER_SEPARATOR +} + +/// Split the contents of one bracketed value into `(first token, rest)` entries. +fn nested_entries<'a>(tokens: &[&'a HeaderToken], depth: i64) -> Vec> { + let mut groups: Vec> = vec![Vec::new()]; + for token in tokens { + if token.depth < depth { + continue; + } + if token.depth == depth && is_separator(token) { + groups.push(Vec::new()); + continue; + } + if let Some(group) = groups.last_mut() { + group.push(token); + } + } + groups + .into_iter() + .filter(|group| group.first().is_some_and(|first| is_value_kind(first.kind))) + .map(|group| (group[0], group[1..].to_vec())) + .collect() +} + +/// Top-level `key value` entries keyed by their first word. +fn header_entries(tokens: &[HeaderToken]) -> Vec<(String, Vec<&HeaderToken>)> { + let all: Vec<&HeaderToken> = tokens.iter().collect(); + nested_entries(&all, 0) + .into_iter() + .filter(|(first, _)| first.kind == HeaderTokenKind::Word) + .map(|(first, rest)| { + let mut entry = vec![first]; + entry.extend(rest); + (first.value.clone(), entry) + }) + .collect() +} + +/// Whether a word or string token spells a name, ignoring case. +pub(crate) fn is_value(token: &HeaderToken, value: &str) -> bool { + is_value_kind(token.kind) && token.value.to_lowercase() == value.to_lowercase() +} + +/// Replace one word or string value, keeping string quoting. +fn rename_value_token( + text: &[char], + token: &HeaderToken, + new_value: &str, + kind: EditKind, +) -> TextEdit { + let replacement = if token.kind == HeaderTokenKind::Word { + new_value.to_owned() + } else { + let quote = text[token.start]; + format!("{quote}{new_value}{quote}") + }; + text_edit(text, (token.start, token.end), replacement, kind) +} + +/// `(input name token, value tokens)` for every `cursor_inputs` entry. +fn cursor_input_tokens(tokens: &[HeaderToken]) -> Vec> { + header_entries(tokens) + .into_iter() + .find(|(key, _)| key == CURSOR_INPUTS_KEY) + .map(|(_, entry)| nested_entries(&entry[1..], 1)) + .unwrap_or_default() +} + +/// The target and field value tokens of every relationships audit. +fn relationship_tokens(tokens: &[HeaderToken]) -> Vec> { + let mut found: Vec> = Vec::new(); + for (index, token) in tokens.iter().enumerate() { + if !(token.kind == HeaderTokenKind::Word + && token.value == RELATIONSHIPS_AUDIT + && index + 1 < tokens.len() + && tokens[index + 1].value == HEADER_OPEN_PAREN) + { + continue; + } + let depth = tokens[index + 1].depth + 1; + let mut to_index: Option = None; + let mut field_index: Option = None; + let mut cursor = index + 2; + while cursor < tokens.len() && tokens[cursor].depth >= depth { + let candidate = &tokens[cursor]; + if candidate.depth == depth + && candidate.kind == HeaderTokenKind::Word + && cursor + 1 < tokens.len() + { + if candidate.value == RELATIONSHIPS_TO_KEY { + to_index = Some(cursor + 1); + } else if candidate.value == RELATIONSHIPS_FIELD_KEY { + field_index = Some(cursor + 1); + } + } + cursor += 1; + } + found.push(relationship(tokens, to_index, field_index)); + } + found +} + +fn relationship( + tokens: &[HeaderToken], + to_index: Option, + field_index: Option, +) -> RelationshipTokens<'_> { + let field = field_index.map(|index| &tokens[index]); + let Some(to_index) = to_index else { + return RelationshipTokens { + target: None, + field, + called: false, + }; + }; + let called = tokens[to_index].value == REF_FUNCTION + && to_index + 2 < tokens.len() + && tokens[to_index + 1].value == HEADER_OPEN_PAREN + && tokens[to_index + 2].kind == HeaderTokenKind::String; + RelationshipTokens { + target: Some(if called { + &tokens[to_index + 2] + } else { + &tokens[to_index] + }), + field, + called, + } +} + +/// Rename a column in column-valued config such as `cursor` or `unique_key`. +pub(crate) fn column_config_edits( + text: &[char], + tokens: &[HeaderToken], + old: &str, + new: &str, +) -> Vec { + let mut edits: Vec = Vec::new(); + for (key, entry) in header_entries(tokens) { + if !COLUMN_VALUED_CONFIG_KEYS.contains(&key.as_str()) { + continue; + } + for token in &entry[1..] { + if is_value(token, old) { + edits.push(rename_value_token(text, token, new, EditKind::Header)); + } + } + } + edits +} + +/// Rename a `columns` entry and add `migrate_from` if needed; `None` without an entry. +pub(crate) fn column_entry_edits( + text: &[char], + tokens: &[HeaderToken], + names: (&str, &str), + migrate: bool, +) -> Option> { + let (old, new) = names; + for (key, entry) in header_entries(tokens) { + if key != COLUMNS_KEY { + continue; + } + for (name_token, values) in nested_entries(&entry[1..], 1) { + if !is_value(name_token, old) { + continue; + } + let mut edits = vec![rename_value_token(text, name_token, new, EditKind::Header)]; + if migrate { + edits.push(column_migration_edit(text, name_token, &values, old)); + } + return Some(edits); + } + } + None +} + +/// Declare `new (migrate_from old)` in the columns block, creating it when absent. +pub(crate) fn add_column_entry_edit( + contents: &str, + text: &[char], + tokens: &[HeaderToken], + names: (&str, &str), +) -> Option { + let (old, new) = names; + let declaration = format!("{new} ({MIGRATE_FROM_KEY} {old})"); + for (key, entry) in header_entries(tokens) { + if key != COLUMNS_KEY || entry.len() < HEADER_KEY_AND_VALUE_TOKENS { + continue; + } + let opener = entry[1]; + let indentation = line_indent(text, entry[0].start); + return Some( + text_edit( + text, + (opener.end, opener.end), + format!("\n{indentation}{HEADER_INDENT}{declaration},"), + EditKind::Migration, + ) + .with_display(Some(String::new()), Some(declaration)), + ); + } + insert_header_entry_edit( + contents, + text, + &format!("{COLUMNS_KEY} ({declaration})"), + &declaration, + ) +} + +/// Insert one top-level entry as the first MODEL header line. +pub(crate) fn insert_header_entry_edit( + contents: &str, + text: &[char], + entry: &str, + display: &str, +) -> Option { + let (start, end) = model_header_body_span(contents)?; + let body: Vec = text[start..end].to_vec(); + let replacement = if starts_with(&body, 0, "\n") { + let first_line: Vec = match find(&body, "\n", 1) { + Some(line_end) => body[1..line_end].to_vec(), + None => body[1..].to_vec(), + }; + let indent_length = first_line + .iter() + .take_while(|character| is_python_space(**character)) + .count(); + let indentation: String = if indent_length == 0 { + HEADER_INDENT.to_owned() + } else { + first_line[..indent_length].iter().collect() + }; + format!("\n{indentation}{entry},") + } else if body.iter().any(|character| !is_python_space(*character)) { + format!("{entry}, ") + } else { + format!("\n{HEADER_INDENT}{entry},\n") + }; + Some( + text_edit(text, (start, start), replacement, EditKind::Migration) + .with_display(Some(String::new()), Some(display.to_owned())), + ) +} + +/// Header offsets that still mention a word outside descriptions. +pub(crate) fn unhandled_word_offsets( + text: &[char], + tokens: &[HeaderToken], + word: &str, + handled: &[usize], +) -> Vec { + let mut offsets: Vec = Vec::new(); + let mut previous: Option<&HeaderToken> = None; + for token in tokens { + let described = previous.is_some_and(|item| item.value == HEADER_DESCRIPTION_KEY); + previous = Some(token); + if token.kind == HeaderTokenKind::Symbol || described || handled.contains(&token.start) { + continue; + } + let token_text: Vec = text[token.start..token.end.min(text.len())].to_vec(); + offsets.extend( + whole_word_offsets(&token_text, word) + .into_iter() + .map(|offset| token.start + offset), + ); + } + offsets +} + +/// Rename a model in `cursor_inputs` keys, bare relationships targets, and quoted SQL. +pub(crate) fn model_name_header_edits( + contents: &str, + text: &[char], + old: &str, + new: &str, +) -> Result, RefactorError> { + Ok(match header_tokens(contents, text)? { + Some(tokens) => model_name_token_edits(text, &tokens, old, new), + None => Vec::new(), + }) +} + +/// Rename a model in the given header tokens. +fn model_name_token_edits( + text: &[char], + tokens: &[HeaderToken], + old: &str, + new: &str, +) -> Vec { + let mut names: Vec<&HeaderToken> = cursor_input_tokens(tokens) + .into_iter() + .map(|(name, _)| name) + .filter(|name| is_value(name, old)) + .collect(); + names.extend( + relationship_tokens(tokens) + .into_iter() + .filter(|relationship| !relationship.called) + .filter_map(|relationship| relationship.target) + .filter(|target| is_value(target, old)), + ); + let mut edits: Vec = names + .into_iter() + .map(|name| rename_value_token(text, name, new, EditKind::Header)) + .collect(); + for token in tokens { + if token.kind != HeaderTokenKind::String { + continue; + } + let raw: Vec = text[token.start..token.end.min(text.len())].to_vec(); + edits.extend( + embedded_ref_spans(&raw, old) + .into_iter() + .map(|(start, end)| { + text_edit( + text, + (token.start + start, token.start + end), + new.to_owned(), + EditKind::Reference, + ) + }), + ); + } + edits +} + +/// Rename an upstream column in `cursor_inputs` values and relationships `field` values. +pub(crate) fn consumer_column_header_edits( + contents: &str, + text: &[char], + upstream: &str, + names: (&str, &str), +) -> Result, RefactorError> { + Ok(match header_tokens(contents, text)? { + Some(tokens) => column_token_edits(text, &tokens, upstream, names), + None => Vec::new(), + }) +} + +/// Rename an upstream column in the given header tokens. +fn column_token_edits( + text: &[char], + tokens: &[HeaderToken], + upstream: &str, + names: (&str, &str), +) -> Vec { + let (old, new) = names; + let mut values: Vec<&HeaderToken> = Vec::new(); + for (name, inputs) in cursor_input_tokens(tokens) { + if is_value(name, upstream) { + values.extend(inputs.into_iter().filter(|value| is_value(value, old))); + } + } + for relationship in relationship_tokens(tokens) { + if let (Some(target), Some(field)) = (relationship.target, relationship.field) + && is_value(target, upstream) + && is_value(field, old) + { + values.push(field); + } + } + values + .into_iter() + .map(|value| rename_value_token(text, value, new, EditKind::Header)) + .collect() +} + +fn column_migration_edit( + text: &[char], + name: &HeaderToken, + values: &[&HeaderToken], + old: &str, +) -> TextEdit { + let declaration = format!("{MIGRATE_FROM_KEY} {old}"); + if let Some(opener) = values + .first() + .filter(|first| first.value == HEADER_OPEN_PAREN) + { + let has_metadata = values.len() > PARENTHESIZED_EMPTY_TOKENS; + let replacement = if has_metadata { + format!("{declaration}, ") + } else { + declaration.clone() + }; + return text_edit( + text, + (opener.end, opener.end), + replacement, + EditKind::Migration, + ) + .with_display(Some(String::new()), Some(declaration)); + } + text_edit( + text, + (name.end, name.end), + format!(" ({declaration})"), + EditKind::Migration, + ) + .with_display(Some(String::new()), Some(declaration)) +} + +fn line_indent(text: &[char], offset: usize) -> String { + let line_start = rfind(text, "\n", 0, offset).map_or(0, |index| index + 1); + let prefix: Vec = text[line_start..offset].to_vec(); + if prefix.iter().all(|character| is_python_space(*character)) { + prefix.into_iter().collect() + } else { + HEADER_INDENT.to_owned() + } +} + +/// Code-point spans of every `SCHEMA (...)` header body, as `SCHEMA_STATEMENT_PATTERN.finditer`. +fn schema_header_spans(contents: &str, text: &[char], python: PythonText) -> Vec<(usize, usize)> { + let mut byte_offsets: Vec = contents.char_indices().map(|(offset, _)| offset).collect(); + byte_offsets.push(contents.len()); + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut index = 0; + while index < text.len() { + let bounded = index == 0 || !is_python_word(python, text[index - 1]); + if bounded + && starts_with(text, index, SCHEMA_KEYWORD) + && let Some((start, end, _)) = + match_statement_header(contents, byte_offsets[index], SCHEMA_KEYWORD) + { + let start = byte_to_char(&byte_offsets, start); + let end = byte_to_char(&byte_offsets, end); + spans.push((start, end)); + index = statement_end(text, end); + continue; + } + index += 1; + } + spans +} + +/// The end of `\)\s*;` after a header body ending at `end`. +fn statement_end(text: &[char], end: usize) -> usize { + let mut index = end + 1; + while text.get(index).copied().is_some_and(is_python_space) { + index += 1; + } + index + 1 +} + +fn byte_to_char(byte_offsets: &[usize], byte: usize) -> usize { + byte_offsets.partition_point(|offset| *offset < byte) +} + +/// Rename a model in every SCHEMA header of a declaration file. +pub(crate) fn schema_model_name_edits( + contents: &str, + text: &[char], + names: (&str, &str), + python: PythonText, +) -> Result, RefactorError> { + let mut edits: Vec = Vec::new(); + for span in schema_header_spans(contents, text, python) { + edits.extend(model_name_token_edits( + text, + &span_tokens(text, span)?, + names.0, + names.1, + )); + } + Ok(edits) +} + +/// Rename relationships `field` values pointing at a renamed column of a model. +pub(crate) fn schema_column_edits( + contents: &str, + text: &[char], + column: (&str, (&str, &str)), + python: PythonText, +) -> Result, RefactorError> { + let (upstream, names) = column; + let mut edits: Vec = Vec::new(); + for span in schema_header_spans(contents, text, python) { + edits.extend(column_token_edits( + text, + &span_tokens(text, span)?, + upstream, + names, + )); + } + Ok(edits) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/mod.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/mod.rs new file mode 100644 index 0000000000..d2a38e3d3f --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod header_edits; +pub(crate) mod text_edits; +pub(crate) mod yaml_edits; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/text_edits.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/text_edits.rs new file mode 100644 index 0000000000..8af141714f --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/text_edits.rs @@ -0,0 +1,299 @@ +//! Offsets, positions, and edit application for authored text, as `text_edits.py` does. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::refactoring::_helpers::scanning::chars::{ + count_before, is_identifier_character, rfind, slice, starts_with_ignoring_case, +}; +use crate::refactoring::constants::EDIT_ERROR_CODE; +use crate::refactoring::errors::{RefactorError, RefactorErrorKind}; +use crate::refactoring::models::{ + EditKind, FileChange, ManualLocation, MigrationDeclaration, RefactorPlan, RefactorRequest, + TextEdit, +}; + +/// Edits and findings from one planning step, merged into a plan at the end. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct RefactorParts { + pub(crate) edits: Vec<(String, TextEdit)>, + pub(crate) manual: Vec, + pub(crate) blocking: Vec, + pub(crate) migrations: Vec, + pub(crate) cascaded: Vec, + pub(crate) moves: Vec<(String, String)>, +} + +/// The 1-based line and column of an offset. +pub(crate) fn line_column(text: &[char], offset: usize) -> (usize, usize) { + let line_start = rfind(text, "\n", 0, offset).map_or(0, |index| index + 1); + ( + count_before(text, '\n', offset) + 1, + offset - line_start + 1, + ) +} + +/// One edit with its position; `before` and `after` default as Python's do. +pub(crate) fn text_edit( + text: &[char], + span: (usize, usize), + replacement: String, + kind: EditKind, +) -> TextEdit { + let (start, end) = span; + let (line, column) = line_column(text, start); + TextEdit { + start, + end, + before: slice(text, start, end), + after: python_strip(&replacement).to_owned(), + replacement, + kind, + line, + column, + } +} + +/// A manual location at an optional offset. +pub(crate) fn manual_at( + path: &str, + text: &[char], + offset: Option, + reason: String, +) -> ManualLocation { + let (line, column) = match offset { + Some(offset) => { + let (line, column) = line_column(text, offset); + (Some(line), Some(column)) + } + None => (None, None), + }; + ManualLocation { + path: path.to_owned(), + line, + column, + reason, + } +} + +/// Apply non-overlapping edits back to front; identical duplicates apply once. +pub(crate) fn apply_text_edits( + text: &[char], + edits: &[TextEdit], +) -> Result, RefactorError> { + let mut unique: Vec<&TextEdit> = Vec::new(); + for edit in edits { + let key = (edit.start, edit.end, &edit.replacement); + match unique + .iter() + .position(|item| (item.start, item.end, &item.replacement) == key) + { + Some(index) => unique[index] = edit, + None => unique.push(edit), + } + } + unique.sort_by_key(|edit| std::cmp::Reverse((edit.start, edit.end))); + let mut result: Vec = text.to_vec(); + let mut previous_start = text.len() + 1; + for edit in unique { + if edit.end > previous_start { + return Err(RefactorError { + kind: RefactorErrorKind::Edit, + code: EDIT_ERROR_CODE.to_owned(), + message: format!("overlapping refactoring edits at offset {}", edit.start), + help: None, + }); + } + let end = edit.end.min(result.len()); + let start = edit.start.min(end); + result.splice(start..end, edit.replacement.chars()); + previous_start = edit.start; + } + Ok(result) +} + +/// Concatenate planning parts in order; moves are not merged, as in Python. +pub(crate) fn merge_parts(parts: Vec) -> RefactorParts { + let mut merged = RefactorParts::default(); + for part in parts { + merged.edits.extend(part.edits); + merged.manual.extend(part.manual); + merged.blocking.extend(part.blocking); + merged.migrations.extend(part.migrations); + merged.cascaded.extend(part.cascaded); + } + merged +} + +/// Attach a file path to each edit. +pub(crate) fn path_edits(path: &str, edits: Vec) -> Vec<(String, TextEdit)> { + edits + .into_iter() + .map(|edit| (path.to_owned(), edit)) + .collect() +} + +/// Group edits by file, move files, and order the result by final path. +pub(crate) fn file_changes( + edits: Vec<(String, TextEdit)>, + moves: &[(String, String)], +) -> Vec { + let mut grouped: Vec<(String, Vec)> = Vec::new(); + for (path, _) in moves { + if !grouped.iter().any(|(item, _)| item == path) { + grouped.push((path.clone(), Vec::new())); + } + } + for (path, edit) in edits { + match grouped.iter_mut().find(|(item, _)| *item == path) { + Some((_, items)) => items.push(edit), + None => grouped.push((path, vec![edit])), + } + } + let mut changes: Vec = grouped + .into_iter() + .map(|(path, items)| FileChange { + path: moved_path(moves, &path), + original_path: path, + edits: items, + }) + .collect(); + changes.sort_by(|left, right| left.path.cmp(&right.path)); + changes +} + +/// `moves.get(path, path)` over moves where a later entry for a path wins, as in a dict. +fn moved_path(moves: &[(String, String)], path: &str) -> String { + moves + .iter() + .rev() + .find(|(source, _)| source == path) + .map_or_else(|| path.to_owned(), |(_, destination)| destination.clone()) +} + +/// Assemble a plan from merged parts; help applies only when manual locations exist. +pub(crate) fn build_plan( + request: RefactorRequest, + parts: RefactorParts, + moves: &[(String, String)], + outcome: (Vec<(String, String, String)>, Option<&str>), +) -> RefactorPlan { + let (renamed_columns, help) = outcome; + let help = (!parts.manual.is_empty()) + .then_some(help) + .flatten() + .map(str::to_owned); + RefactorPlan { + request, + changes: file_changes(parts.edits, moves), + manual: parts.manual, + blocking: parts.blocking, + migrations: parts.migrations, + renamed_columns, + help, + } +} + +/// Unquoted code identifiers matching `names`, as `identifier_sites` finds them. +pub(crate) fn identifier_sites(text: &[char], names: &[String]) -> Vec<(usize, usize, String)> { + if names.is_empty() { + return Vec::new(); + } + let mut lowered: Vec<(String, String)> = Vec::new(); + for name in names { + let key = name.to_lowercase(); + match lowered.iter_mut().find(|(item, _)| *item == key) { + Some(entry) => entry.1 = name.clone(), + None => lowered.push((key, name.clone())), + } + } + let mut ordered: Vec> = lowered + .iter() + .map(|(key, _)| key.chars().collect()) + .collect(); + ordered.sort_by(|left, right| right.len().cmp(&left.len()).then_with(|| left.cmp(right))); + let mut sites: Vec<(usize, usize, String)> = Vec::new(); + let mut index = 0; + while index < text.len() { + if let Some(end) = non_code_end(text, index) { + index = end; + continue; + } + let preceded = index > 0 && is_identifier_character(text[index - 1]); + let found = (!preceded) + .then(|| { + ordered.iter().find(|name| { + starts_with_ignoring_case(text, index, name) + && !text + .get(index + name.len()) + .is_some_and(|next| is_identifier_character(*next)) + }) + }) + .flatten(); + match found { + Some(name) => { + let end = index + name.len(); + let token: String = slice(text, index, end).to_lowercase(); + let canonical = lowered + .iter() + .find(|(key, _)| *key == token) + .map_or_else(|| token.clone(), |(_, name)| name.clone()); + sites.push((index, end, canonical)); + index = end; + } + None => index += 1, + } + } + sites +} + +/// The end of `NON_CODE_PATTERN` matched at `index`: a comment or a quoted string. +fn non_code_end(text: &[char], index: usize) -> Option { + let at = |offset: usize| text.get(index + offset).copied(); + match (at(0), at(1)) { + (Some('-'), Some('-')) => Some( + (index + 2..text.len()) + .find(|position| text[*position] == '\n') + .map_or(text.len(), |position| position + 1), + ), + (Some('/'), Some('*')) => Some( + (index + 2..text.len().saturating_sub(1)) + .find(|position| text[*position] == '*' && text[*position + 1] == '/') + .map_or(text.len(), |position| position + 2), + ), + (Some(quote @ ('\'' | '"')), _) => { + let mut position = index + 1; + while position < text.len() { + if text[position] == quote { + if text.get(position + 1) == Some("e) { + position += 2; + continue; + } + return Some(position + 1); + } + position += 1; + } + Some(text.len()) + } + _ => None, + } +} + +/// Offsets of a case-insensitive whole identifier word anywhere in `text`. +pub(crate) fn whole_word_offsets(text: &[char], word: &str) -> Vec { + let word: Vec = word.chars().collect(); + let mut offsets: Vec = Vec::new(); + let mut index = 0; + while index + word.len() <= text.len() { + let preceded = index > 0 && is_identifier_character(text[index - 1]); + let followed = text + .get(index + word.len()) + .is_some_and(|next| is_identifier_character(*next)); + if !preceded && !followed && starts_with_ignoring_case(text, index, &word) { + offsets.push(index); + index += word.len().max(1); + continue; + } + index += 1; + } + offsets +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/yaml_edits.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/yaml_edits.rs new file mode 100644 index 0000000000..9a2dc4ce40 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/edits/yaml_edits.rs @@ -0,0 +1,207 @@ +//! Model and column references in YAML declarations, as `yaml_edits.py` finds them. + +use sqlbuild_config::errors::ConfigErrorKind; +use sqlbuild_config::models::{ComposedYaml, ComposedYamlContent, ComposedYamlNode}; +use sqlbuild_config::yaml::main::compose_marks::compose_marks; + +use crate::refactoring::_helpers::edits::text_edits::{path_edits, text_edit}; +use crate::refactoring::_helpers::scanning::chars::{chars, find}; +use crate::refactoring::_helpers::scanning::sql_sites::embedded_ref_spans; +use crate::refactoring::constants::{ + RELATIONSHIPS_AUDIT, RELATIONSHIPS_FIELD_KEY, RELATIONSHIPS_TO_KEY, +}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{DiscoveredFile, EditKind, TextEdit}; + +/// The `to` and `field` scalars of one relationships audit. +struct YamlRelationship<'a> { + target: Option<&'a ComposedYamlNode>, + field: Option<&'a ComposedYamlNode>, +} + +/// One YAML file's text and composed nodes, or `None` where PyYAML raises a `YAMLError`. +struct ComposedFile<'a> { + file: &'a DiscoveredFile, + text: Vec, + document: ComposedYaml, +} + +fn compose(file: &DiscoveredFile) -> Result>, RefactorError> { + match compose_marks(&file.contents) { + Ok(document) if document.root.is_some() => Ok(Some(ComposedFile { + file, + text: chars(&file.contents), + document, + })), + Ok(_) => Ok(None), + Err(error) if error.kind == ConfigErrorKind::Unsupported => Err(RefactorError::internal( + format!("{}: {}", file.path, error.message), + )), + Err(_) => Ok(None), + } +} + +impl ComposedFile<'_> { + fn node(&self, id: usize) -> &ComposedYamlNode { + &self.document.nodes[id] + } + + fn scalar_value(node: &ComposedYamlNode) -> Option<&str> { + node.content.as_scalar() + } + + fn raw(&self, node: &ComposedYamlNode) -> Vec { + let end = node.end.min(self.text.len()); + self.text[node.start.min(end)..end].to_vec() + } + + /// Every scalar reached through sequence items and mapping values, in document order. + fn scalars(&self, id: usize) -> Vec { + match &self.node(id).content { + ComposedYamlContent::Scalar(_) => vec![id], + ComposedYamlContent::Sequence(items) => { + items.iter().flat_map(|item| self.scalars(*item)).collect() + } + ComposedYamlContent::Mapping(entries) => entries + .iter() + .flat_map(|(_, value)| self.scalars(*value)) + .collect(), + } + } + + fn relationships(&self, id: usize) -> Vec> { + let mut found: Vec> = Vec::new(); + match &self.node(id).content { + ComposedYamlContent::Mapping(entries) => { + for (key, value) in entries { + let is_audit = Self::scalar_value(self.node(*key)) == Some(RELATIONSHIPS_AUDIT); + if is_audit + && let ComposedYamlContent::Mapping(audit) = &self.node(*value).content + { + found.push(self.relationship(audit)); + } + found.extend(self.relationships(*value)); + } + } + ComposedYamlContent::Sequence(items) => { + for item in items { + found.extend(self.relationships(*item)); + } + } + ComposedYamlContent::Scalar(_) => {} + } + found + } + + fn relationship(&self, entries: &[(usize, usize)]) -> YamlRelationship<'_> { + let mut target: Option<&ComposedYamlNode> = None; + let mut field: Option<&ComposedYamlNode> = None; + for (key, value) in entries { + let value_node = self.node(*value); + if Self::scalar_value(value_node).is_none() { + continue; + } + match Self::scalar_value(self.node(*key)) { + Some(RELATIONSHIPS_TO_KEY) => target = Some(value_node), + Some(RELATIONSHIPS_FIELD_KEY) => field = Some(value_node), + _ => {} + } + } + YamlRelationship { target, field } + } + + fn rename_scalar(&self, node: &ComposedYamlNode, new: &str, kind: EditKind) -> TextEdit { + let value = Self::scalar_value(node).unwrap_or_default(); + let raw: Vec = chars(&self.raw(node).iter().collect::().to_lowercase()); + let offset = find(&raw, &value.to_lowercase(), 0).unwrap_or(0); + let start = node.start + offset; + text_edit( + &self.text, + (start, start + value.chars().count()), + new.to_owned(), + kind, + ) + } + + fn targets(&self, relationship: &YamlRelationship<'_>, model: &str) -> bool { + let Some(target) = relationship.target else { + return false; + }; + Self::scalar_value(target) == Some(model) + || !embedded_ref_spans(&self.raw(target), model).is_empty() + } +} + +/// Rename `__ref` calls in YAML strings and bare relationships targets naming a model. +pub(crate) fn yaml_model_edits( + files: &[DiscoveredFile], + old: &str, + new: &str, +) -> Result, RefactorError> { + let mut edits: Vec<(String, TextEdit)> = Vec::new(); + for file in files { + let Some(composed) = compose(file)? else { + continue; + }; + let root = composed.document.root.unwrap_or_default(); + let scalars: Vec = composed.scalars(root); + let mut found: Vec = Vec::new(); + for id in scalars { + let node = composed.node(id); + found.extend( + embedded_ref_spans(&composed.raw(node), old) + .into_iter() + .map(|(start, end)| { + text_edit( + &composed.text, + (node.start + start, node.start + end), + new.to_owned(), + EditKind::Reference, + ) + }), + ); + } + found.extend( + composed + .relationships(root) + .into_iter() + .filter_map(|relationship| relationship.target) + .filter(|target| ComposedFile::scalar_value(target) == Some(old)) + .map(|target| composed.rename_scalar(target, new, EditKind::Reference)), + ); + edits.extend(path_edits(&composed.file.path, found)); + } + Ok(edits) +} + +/// Rename relationships `field` values that point at a renamed column of a model. +pub(crate) fn yaml_column_edits( + files: &[DiscoveredFile], + model: &str, + old: &str, + new: &str, +) -> Result, RefactorError> { + let mut edits: Vec<(String, TextEdit)> = Vec::new(); + for file in files { + let Some(composed) = compose(file)? else { + continue; + }; + let root = composed.document.root.unwrap_or_default(); + let found: Vec = composed + .relationships(root) + .iter() + .filter(|relationship| { + relationship.field.is_some_and(|field| { + ComposedFile::scalar_value(field) + .unwrap_or_default() + .to_lowercase() + == old.to_lowercase() + }) && composed.targets(relationship, model) + }) + .filter_map(|relationship| relationship.field) + .map(|field| composed.rename_scalar(field, new, EditKind::Column)) + .collect(); + edits.extend(path_edits(&composed.file.path, found)); + } + Ok(edits) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/files/mod.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/files/mod.rs new file mode 100644 index 0000000000..ef262d2e4d --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/files/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod paths; +pub(crate) mod project_files; +pub(crate) mod workspace; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/files/paths.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/files/paths.rs new file mode 100644 index 0000000000..a001124bc7 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/files/paths.rs @@ -0,0 +1,96 @@ +//! `pathlib` path semantics the refactoring planner relies on. + +use std::path::{Component, Path, PathBuf}; + +use crate::refactoring::constants::CURRENT_DIRECTORY; + +/// Non-strict `Path.resolve()`: symlinks resolve while the path exists, then lexically. +pub(crate) fn resolve(path: &Path) -> PathBuf { + let mut resolved = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + resolved.pop(); + } + other => { + resolved.push(other.as_os_str()); + if let Ok(canonical) = std::fs::canonicalize(&resolved) { + resolved = canonical; + } + } + } + } + resolved +} + +/// A relative path as `PurePath.as_posix()` spells it, or `None` when not relative. +pub(crate) fn relative_posix(path: &Path, base: &Path) -> Option { + let Ok(relative) = path.strip_prefix(base) else { + return None; + }; + let parts: Vec = relative + .components() + .map(|component| component.as_os_str().to_string_lossy().into_owned()) + .collect(); + Some(if parts.is_empty() { + ".".to_owned() + } else { + parts.join("/") + }) +} + +/// `PurePosixPath(path).name`. +pub(crate) fn name(path: &str) -> &str { + path.trim_end_matches('/') + .rsplit('/') + .next() + .unwrap_or_default() +} + +/// `PurePosixPath(path).parent`, as a string (`.` for a bare name). +pub(crate) fn parent(path: &str) -> String { + let parts: Vec<&str> = path + .split('/') + .filter(|part| !part.is_empty() && *part != CURRENT_DIRECTORY) + .collect(); + if parts.len() <= 1 { + return ".".to_owned(); + } + parts[..parts.len() - 1].join("/") +} + +/// `PurePath.suffix`: the final component's last dot suffix, unless the dot starts the name. +pub(crate) fn suffix(name: &str) -> &str { + match name.rfind('.') { + Some(index) if index > 0 && index < name.len() - 1 => &name[index..], + _ => "", + } +} + +/// `PurePosixPath(path).stem`. +pub(crate) fn stem(path: &str) -> &str { + let name = name(path); + let suffix = suffix(name); + &name[..name.len() - suffix.len()] +} + +/// `PurePosixPath(path).with_name(name).as_posix()`. +pub(crate) fn with_name(path: &str, new_name: &str) -> String { + match parent(path).as_str() { + "." => new_name.to_owned(), + parent => format!("{parent}/{new_name}"), + } +} + +/// `project_dir / PurePosixPath(relative)`. +pub(crate) fn join(project_dir: &Path, relative: &str) -> PathBuf { + let mut path = project_dir.to_path_buf(); + for part in relative + .split('/') + .filter(|part| !part.is_empty() && *part != CURRENT_DIRECTORY) + { + path.push(part); + } + path +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/files/project_files.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/files/project_files.rs new file mode 100644 index 0000000000..677f650b4a --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/files/project_files.rs @@ -0,0 +1,95 @@ +//! Authored files and SQL bodies a refactoring edits, as `project_files.py` collects them. + +use std::collections::BTreeMap; + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::refactoring::_helpers::scanning::chars::{chars, find}; +use crate::refactoring::models::{DiscoveredFile, RefactorFacts, SqlFileRole}; + +/// One authored SQL file, the role discovery gave it, and its code points. +#[derive(Clone, Debug)] +pub(crate) struct ProjectSqlFile { + pub(crate) path: String, + pub(crate) contents: String, + pub(crate) text: Vec, + pub(crate) role: SqlFileRole, +} + +/// One SQL body inside an authored non-model file. +#[derive(Clone, Debug)] +pub(crate) struct AuthoredBody { + pub(crate) path: String, + pub(crate) role: SqlFileRole, + pub(crate) text: Vec, + pub(crate) contents: Vec, + pub(crate) start: usize, + pub(crate) body: String, +} + +/// Discovered SQL files with authored contents, sorted by path; the first of a path wins. +pub(crate) fn project_sql_files(facts: &RefactorFacts) -> Vec { + let mut files: BTreeMap = BTreeMap::new(); + let models = facts.model_files.iter().map(|file| { + ( + SqlFileRole::Model, + file.path.as_str(), + file.contents.as_str(), + ) + }); + let authored = facts + .authored_files + .iter() + .map(|file| (file.role, file.path.as_str(), file.contents.as_str())); + for (role, path, contents) in models.chain(authored) { + files + .entry(path.to_owned()) + .or_insert_with(|| ProjectSqlFile { + path: path.to_owned(), + contents: contents.to_owned(), + text: chars(contents), + role, + }); + } + files.into_values().collect() +} + +/// Every source and seed YAML declaration file, the last contents of a path winning, by path. +pub(crate) fn yaml_files(facts: &RefactorFacts) -> Vec { + let mut files: BTreeMap = BTreeMap::new(); + for file in &facts.yaml_files { + files.insert(file.path.clone(), file.contents.clone()); + } + files + .into_iter() + .map(|(path, contents)| DiscoveredFile { path, contents }) + .collect() +} + +/// Every non-model SQL body located in its file, in file order. +pub(crate) fn authored_bodies(facts: &RefactorFacts) -> Vec { + let mut bodies: Vec = Vec::new(); + for file in &facts.authored_files { + let contents = chars(&file.contents); + let mut cursor = 0; + for body in &file.texts { + if python_strip(body).is_empty() { + continue; + } + let Some(start) = find(&contents, body, cursor) else { + continue; + }; + let text = chars(body); + cursor = start + text.len(); + bodies.push(AuthoredBody { + path: file.path.clone(), + role: file.role, + text, + contents: contents.clone(), + start, + body: body.clone(), + }); + } + } + bodies +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/files/workspace.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/files/workspace.rs new file mode 100644 index 0000000000..4751994bda --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/files/workspace.rs @@ -0,0 +1,343 @@ +//! Stage edits in a scratch copy, then commit with rollback, as `workspace.py` does. + +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use sqlbuild_core::text::main::decode_python_text::decode_python_text; + +use crate::refactoring::_helpers::edits::text_edits::apply_text_edits; +use crate::refactoring::_helpers::files::paths::{join, suffix}; +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::constants::{ + COPIED_PROJECT_SUFFIXES, HIDDEN_PREFIX, IGNORED_PROJECT_DIRECTORIES, PYCACHE_DIRECTORY, + WRITE_ERROR_CODE, +}; +use crate::refactoring::errors::{RefactorError, RefactorErrorKind}; +use crate::refactoring::models::FileChange; +use crate::refactoring::types::Originals; + +fn io_error(path: &Path, error: impl std::fmt::Display) -> RefactorError { + RefactorError { + kind: RefactorErrorKind::Io, + code: String::new(), + message: format!("{}: {error}", path.display()), + help: None, + } +} + +/// Copy every file compilation can read, skipping build output and hidden directories. +pub(crate) fn copy_project_inputs( + project_dir: &Path, + staging_dir: &Path, +) -> Result { + copy_directory(project_dir, project_dir, staging_dir) +} + +fn copy_directory(root: &Path, current: &Path, staging_dir: &Path) -> Result { + let mut entries: Vec = fs::read_dir(current) + .map_err(|error| io_error(current, error))? + .filter_map(Result::ok) + .collect(); + entries.sort_by_key(fs::DirEntry::file_name); + let mut copied = 0; + let mut directories: Vec = Vec::new(); + for entry in entries { + let path = entry.path(); + let name = entry.file_name().to_string_lossy().into_owned(); + let is_link = entry.file_type().is_ok_and(|kind| kind.is_symlink()); + if path.is_dir() { + let ignored = name.starts_with(HIDDEN_PREFIX) + || (current == root && IGNORED_PROJECT_DIRECTORIES.contains(&name.as_str())) + || name == PYCACHE_DIRECTORY; + if !ignored && !is_link { + directories.push(path); + } + continue; + } + if !COPIED_PROJECT_SUFFIXES.contains(&suffix(&name)) || name.starts_with(HIDDEN_PREFIX) { + continue; + } + let relative = path + .strip_prefix(root) + .map_err(|error| io_error(&path, error))?; + let destination = staging_dir.join(relative); + if let Some(parent) = destination.parent() { + fs::create_dir_all(parent).map_err(|error| io_error(parent, error))?; + } + fs::copy(&path, &destination).map_err(|error| io_error(&path, error))?; + copied += 1; + } + for directory in directories { + copied += copy_directory(root, &directory, staging_dir)?; + } + Ok(copied) +} + +/// `Path.read_text(encoding="utf-8")`: strict UTF-8 with universal newlines. +fn read_text(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|error| io_error(path, error))?; + decode_python_text(&bytes).map_err(|_| io_error(path, "'utf-8' codec can't decode the file")) +} + +/// The current text of every file a plan changes. +pub(crate) fn read_originals( + project_dir: &Path, + changes: &[FileChange], +) -> Result { + let mut originals: Originals = Vec::new(); + for change in changes { + let text = read_text(&join(project_dir, &change.original_path))?; + match originals + .iter_mut() + .find(|(path, _)| *path == change.original_path) + { + Some(entry) => entry.1 = text, + None => originals.push((change.original_path.clone(), text)), + } + } + Ok(originals) +} + +fn original<'a>(originals: &'a Originals, path: &str) -> &'a str { + originals + .iter() + .find(|(item, _)| item == path) + .map(|(_, text)| text.as_str()) + .unwrap_or_default() +} + +/// The new text of every changed file, keyed by its final path. +fn edited_contents( + originals: &Originals, + changes: &[FileChange], +) -> Result, RefactorError> { + changes + .iter() + .map(|change| { + let text = chars(original(originals, &change.original_path)); + let edited: String = apply_text_edits(&text, &change.edits)? + .into_iter() + .collect(); + Ok((change.path.clone(), edited)) + }) + .collect() +} + +fn contents_for<'a>(contents: &'a [(String, String)], path: &str) -> &'a str { + contents + .iter() + .rev() + .find(|(item, _)| item == path) + .map(|(_, text)| text.as_str()) + .unwrap_or_default() +} + +/// `Path.write_text`: text mode translates line breaks to the platform's. +fn write_text(path: &Path, contents: &str) -> Result<(), RefactorError> { + let translated = if cfg!(windows) { + contents.replace('\n', "\r\n") + } else { + contents.to_owned() + }; + fs::write(path, translated).map_err(|error| io_error(path, error)) +} + +/// Apply a plan to the scratch copy. +pub(crate) fn write_staged_changes( + staging_dir: &Path, + originals: &Originals, + changes: &[FileChange], +) -> Result<(), RefactorError> { + let contents = edited_contents(originals, changes)?; + for change in changes.iter().filter(|change| change.moved()) { + let _ = fs::remove_file(join(staging_dir, &change.original_path)); + } + for change in changes { + let target = join(staging_dir, &change.path); + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|error| io_error(parent, error))?; + } + write_text(&target, contents_for(&contents, &change.path))?; + } + prune_emptied_directories(staging_dir, changes); + Ok(()) +} + +/// Write a verified plan into the project, restoring every file if any write fails. +pub(crate) fn commit_changes( + project_dir: &Path, + originals: &Originals, + changes: &[FileChange], +) -> Result, RefactorError> { + let current = read_originals(project_dir, changes)?; + let changed: Vec = changed_paths(originals, ¤t); + if !changed.is_empty() || current.len() != originals.len() { + return Err(RefactorError { + kind: RefactorErrorKind::Write, + code: WRITE_ERROR_CODE.to_owned(), + message: format!( + "files changed while the refactoring ran: {}", + changed.join(", ") + ), + help: Some("nothing was written; run the command again".to_owned()), + }); + } + let contents = edited_contents(originals, changes)?; + let (progress, result) = write_all(project_dir, changes, &contents); + if let Err(error) = result { + rollback( + project_dir, + originals, + &progress.written, + &progress.created_directories, + ); + return Err(error); + } + prune_emptied_directories(project_dir, changes); + Ok(progress.written) +} + +/// Original paths whose text on disk differs from the planned original, sorted. +fn changed_paths(originals: &Originals, current: &[(String, String)]) -> Vec { + let mut changed: Vec = originals + .iter() + .filter(|(path, text)| current_text(current, path) != Some(text)) + .map(|(path, _)| path.clone()) + .collect(); + changed.sort(); + changed +} + +fn current_text<'a>(current: &'a [(String, String)], path: &str) -> Option<&'a String> { + current + .iter() + .find(|(item, _)| item == path) + .map(|(_, now)| now) +} + +/// What a commit wrote before it finished or failed. +#[derive(Default)] +struct WriteProgress { + created_directories: Vec, + written: Vec, +} + +fn write_all( + project_dir: &Path, + changes: &[FileChange], + contents: &[(String, String)], +) -> (WriteProgress, Result<(), RefactorError>) { + let mut progress = WriteProgress::default(); + let mut write = || -> Result<(), RefactorError> { + for change in changes { + let target = join(project_dir, &change.path); + if let Some(parent) = target.parent() { + progress + .created_directories + .extend(missing_directories(parent)); + fs::create_dir_all(parent).map_err(|error| io_error(parent, error))?; + } + progress.written.push(target.clone()); + if !target.exists() { + let source = join(project_dir, &change.original_path); + fs::copy(&source, &target).map_err(|error| io_error(&source, error))?; + } + write_atomically(&target, contents_for(contents, &change.path))?; + } + for change in changes.iter().filter(|change| change.moved()) { + let source = join(project_dir, &change.original_path); + fs::remove_file(&source).map_err(|error| io_error(&source, error))?; + } + Ok(()) + }; + let result = write(); + (progress, result) +} + +/// Replace a text file atomically with a temporary file in its folder, keeping its mode. +fn write_atomically(path: &Path, contents: &str) -> Result<(), RefactorError> { + let directory = path.parent().unwrap_or_else(|| Path::new(".")); + let file_name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let permissions = fs::metadata(path) + .map_err(|error| io_error(path, error))? + .permissions(); + let mut temporary = tempfile::Builder::new() + .prefix(&format!(".{file_name}.")) + .tempfile_in(directory) + .map_err(|error| io_error(path, error))?; + fs::set_permissions(temporary.path(), permissions).map_err(|error| io_error(path, error))?; + temporary + .write_all(contents.as_bytes()) + .and_then(|()| temporary.as_file().sync_all()) + .map_err(|error| io_error(path, error))?; + temporary + .persist(path) + .map_err(|error| io_error(path, error.error))?; + Ok(()) +} + +/// Remove folders a move left empty, walking up to the root. +pub(crate) fn prune_emptied_directories(root: &Path, changes: &[FileChange]) { + for change in changes.iter().filter(|change| change.moved()) { + let mut directory = join(root, &change.original_path); + directory.pop(); + while directory != root + && directory.is_dir() + && fs::read_dir(&directory).is_ok_and(|mut entries| entries.next().is_none()) + { + if fs::remove_dir(&directory).is_err() { + break; + } + directory.pop(); + } + } +} + +fn rollback( + project_dir: &Path, + originals: &Originals, + written: &[PathBuf], + created_directories: &[PathBuf], +) { + let original_paths: Vec = originals + .iter() + .map(|(path, _)| join(project_dir, path)) + .collect(); + for path in written { + if !original_paths.contains(path) { + let _ = fs::remove_file(path); + } + } + for (relative, text) in originals { + let restored = join(project_dir, relative); + if restored.exists() { + let _ = write_atomically(&restored, text); + } else { + let _ = write_text(&restored, text); + } + } + for directory in created_directories.iter().rev() { + if directory.is_dir() + && fs::read_dir(directory).is_ok_and(|mut entries| entries.next().is_none()) + { + let _ = fs::remove_dir(directory); + } + } +} + +fn missing_directories(path: &Path) -> Vec { + let mut missing: Vec = Vec::new(); + let mut current = path.to_path_buf(); + while !current.exists() { + missing.push(current.clone()); + if !current.pop() { + break; + } + } + missing.reverse(); + missing +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/mod.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/mod.rs new file mode 100644 index 0000000000..4501beb56f --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod contracts; +pub(crate) mod edits; +pub(crate) mod files; +pub(crate) mod planning; +pub(crate) mod scanning; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_planning.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_planning.rs new file mode 100644 index 0000000000..f7b4c7c165 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_planning.rs @@ -0,0 +1,513 @@ +//! Walk a column rename through every downstream reader, as `column_planning.py` does. + +use std::collections::{BTreeSet, HashSet, VecDeque}; + +use crate::refactoring::_helpers::edits::header_edits::{ + add_column_entry_edit, column_config_edits, column_entry_edits, consumer_column_header_edits, + header_tokens, schema_column_edits, unhandled_word_offsets, +}; +use crate::refactoring::_helpers::edits::text_edits::{ + RefactorParts, manual_at, merge_parts, path_edits, +}; +use crate::refactoring::_helpers::edits::yaml_edits::yaml_column_edits; +use crate::refactoring::_helpers::files::project_files::{ + AuthoredBody, ProjectSqlFile, authored_bodies, project_sql_files, yaml_files, +}; +use crate::refactoring::_helpers::planning::column_references::{ + BodyContext, BodyMapping, ColumnQuery, ResourceColumns, analyze_column, columns_of, + consumer_edits, output_edits, resource_columns, +}; +use crate::refactoring::_helpers::planning::model_planning::{ + find_model, needs_column_migration, validate_identifier, +}; +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::_helpers::scanning::scan_context::ScanContext; +use crate::refactoring::_helpers::scanning::sql_sites::{analysis_sql, model_body}; +use crate::refactoring::constants::{ + EXPECTED_FIXTURE_PREFIX, MIGRATE_FROM_KEY, MODEL_RESOURCE_TYPE, REF_FIXTURE_PREFIX, REF_KIND, + ROOT_SCOPE, +}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{ + DiscoveredFile, ManualLocation, MigrationDeclaration, ModelFacts, RefactorFacts, + RefactorRequest, SqlFileRole, TextEdit, +}; + +/// Everything a column rename reads while it walks downstream models. +pub(crate) struct ColumnRenameContext<'a> { + facts: &'a RefactorFacts, + scan: &'a ScanContext, + owner: &'a ModelFacts, + columns: ResourceColumns, + files: Vec, + bodies: Vec, + yaml_files: Vec, + pub(crate) old: String, + pub(crate) new: String, + cascade: bool, +} + +impl ColumnRenameContext<'_> { + fn names(&self) -> (&str, &str) { + (&self.old, &self.new) + } + + fn file(&self, path: &str) -> Option<&ProjectSqlFile> { + self.files.iter().find(|file| file.path == path) + } + + fn output_names(&self) -> BTreeSet { + columns_of(&self.columns, REF_KIND, &self.owner.name) + .iter() + .map(|name| name.to_lowercase()) + .collect() + } +} + +/// Validate a column rename and collect everything its walk reads. +pub(crate) fn column_rename_context<'a>( + facts: &'a RefactorFacts, + request: &RefactorRequest, + scan: &'a ScanContext, +) -> Result, RefactorError> { + let owner = find_model(&facts.models, &request.model_name)?; + let old = request.column_name.clone().unwrap_or_default(); + let new = request.new_name.clone(); + validate_identifier(&new, "column")?; + let context = ColumnRenameContext { + facts, + scan, + owner, + columns: resource_columns(facts), + files: project_sql_files(facts), + bodies: authored_bodies(facts), + yaml_files: yaml_files(facts), + old, + new, + cascade: request.cascade, + }; + if !context.output_names().contains(&context.old.to_lowercase()) { + return Err(RefactorError::input( + "C956", + format!( + "model:{} has no output column '{}'", + owner.name, context.old + ), + Some("column names match case-insensitively; check the spelling"), + )); + } + if context.old.to_lowercase() == context.new.to_lowercase() { + return Err(RefactorError::input( + "C953", + format!("column {} already has that name", context.old), + None, + )); + } + Ok(context) +} + +/// Every edit of the rename; `cascaded` lists each model whose column is renamed. +pub(crate) fn column_rename_parts( + context: &ColumnRenameContext<'_>, +) -> Result { + let mut parts: Vec = vec![collision(context), owner_output(context)?]; + let mut queue: VecDeque<&ModelFacts> = VecDeque::from([context.owner]); + let mut seen: HashSet = HashSet::new(); + while let Some(model) = queue.pop_front() { + if !seen.insert(model.name.clone()) { + continue; + } + let (consumers, cascaded) = consumers(context, model)?; + let mut declared_edits = + yaml_column_edits(&context.yaml_files, &model.name, &context.old, &context.new)?; + declared_edits.extend(schema_edits(context, model)?); + parts.push(RefactorParts { + cascaded: vec![model.name.clone()], + ..RefactorParts::default() + }); + parts.push(declarations(context, model)?); + parts.push(consumers); + parts.push(authored_body_parts(context, model)?); + parts.push(RefactorParts { + edits: declared_edits, + ..RefactorParts::default() + }); + queue.extend(cascaded); + } + let merged = merge_parts(parts); + let edited: HashSet<&str> = merged.edits.iter().map(|(path, _)| path.as_str()).collect(); + let python = RefactorParts { + manual: context + .facts + .python_locations + .iter() + .filter(|location| !edited.contains(location.path.as_str())) + .cloned() + .collect(), + ..RefactorParts::default() + }; + Ok(merge_parts(vec![merged, python])) +} + +fn schema_edits( + context: &ColumnRenameContext<'_>, + model: &ModelFacts, +) -> Result, RefactorError> { + let mut edits: Vec<(String, TextEdit)> = Vec::new(); + for file in context + .files + .iter() + .filter(|file| file.role == SqlFileRole::Schema) + { + edits.extend(path_edits( + &file.path, + schema_column_edits( + &file.contents, + &file.text, + (&model.name, context.names()), + context.scan.python, + )?, + )); + } + Ok(edits) +} + +fn collision(context: &ColumnRenameContext<'_>) -> RefactorParts { + if !context.output_names().contains(&context.new.to_lowercase()) { + return RefactorParts::default(); + } + RefactorParts { + blocking: vec![ManualLocation { + path: context.owner.path.clone(), + line: None, + column: None, + reason: format!( + "model:{} already has a column named {}", + context.owner.name, context.new + ), + }], + ..RefactorParts::default() + } +} + +fn owner_output(context: &ColumnRenameContext<'_>) -> Result { + let (body, compiled_sql, unmapped) = model_context(context, context.owner); + let Some(body) = body else { + return Ok(RefactorParts { + manual: unmapped, + ..RefactorParts::default() + }); + }; + let scopes = BTreeSet::from([ROOT_SCOPE.to_owned()]); + let facts = analyze_column( + &analysis_sql(&chars(&compiled_sql), context.scan), + &context.facts.dialect, + &context.columns, + &ColumnQuery { + column: &context.old, + target_tables: BTreeSet::new(), + target_ctes: BTreeSet::new(), + output_scopes: scopes.clone(), + }, + )?; + let result = output_edits(&facts, &body, &scopes, context.names()); + let mut manual = result.manual; + if result.edits.is_empty() && manual.is_empty() { + manual.push(ManualLocation { + path: body.path.to_owned(), + line: None, + column: None, + reason: format!( + "cannot find where model:{} produces {}", + context.owner.name, context.old + ), + }); + } + Ok(RefactorParts { + edits: path_edits(body.path, result.edits), + manual, + ..RefactorParts::default() + }) +} + +fn declarations( + context: &ColumnRenameContext<'_>, + model: &ModelFacts, +) -> Result { + let path = &model.path; + let declared = model + .schema_columns + .iter() + .flatten() + .find(|column| column.name.to_lowercase() == context.old.to_lowercase()); + if declared.is_some_and(|column| column.migrate_from) { + return Ok(RefactorParts { + blocking: vec![ManualLocation { + path: path.clone(), + line: None, + column: None, + reason: format!( + "column {} of model:{} still declares migrate_from; build it on every target and remove migrate_from before renaming it again", + context.old, model.name + ), + }], + ..RefactorParts::default() + }); + } + let shared: Vec = declared + .and_then(|column| column.location.as_ref()) + .filter(|location| location.path != *path) + .map(|location| ManualLocation { + path: location.path.clone(), + line: location.line, + column: location.column, + reason: format!( + "column {} of model:{} is declared in a shared schema", + context.old, model.name + ), + }) + .into_iter() + .collect(); + Ok(merge_parts(vec![ + RefactorParts { + manual: shared, + ..RefactorParts::default() + }, + header_parts(context, model)?, + ])) +} + +fn header_parts( + context: &ColumnRenameContext<'_>, + model: &ModelFacts, +) -> Result { + let path = &model.path; + let (contents, text) = context + .file(path) + .map(|file| (file.contents.clone(), file.text.clone())) + .unwrap_or_default(); + let tokens = header_tokens(&contents, &text)?.unwrap_or_default(); + let migrate = needs_column_migration(model); + let (old, new) = context.names(); + let entry_edits = column_entry_edits(&text, &tokens, (old, new), migrate); + let added = if entry_edits.is_none() && migrate { + add_column_entry_edit(&contents, &text, &tokens, (old, new)) + } else { + None + }; + let mut edits = column_config_edits(&text, &tokens, old, new); + edits.extend(entry_edits.unwrap_or_default()); + edits.extend(added); + let handled: Vec = edits.iter().map(|edit| edit.start).collect(); + let manual: Vec = unhandled_word_offsets(&text, &tokens, old, &handled) + .into_iter() + .map(|offset| { + manual_at( + path, + &text, + Some(offset), + format!("the MODEL header mentions {old} here; update it by hand"), + ) + }) + .collect(); + let migrations = if migrate { + vec![MigrationDeclaration { + model_name: model.name.clone(), + declaration: format!("{new} ({MIGRATE_FROM_KEY} {old})"), + reason: "keeps the column's history in place".to_owned(), + }] + } else { + Vec::new() + }; + Ok(RefactorParts { + edits: path_edits(path, edits), + manual, + migrations, + ..RefactorParts::default() + }) +} + +fn consumers<'a>( + context: &ColumnRenameContext<'a>, + model: &ModelFacts, +) -> Result<(RefactorParts, Vec<&'a ModelFacts>), RefactorError> { + let mut parts: Vec = Vec::new(); + let mut cascaded: Vec<&'a ModelFacts> = Vec::new(); + for consumer in &context.facts.models { + if !consumer + .deps + .iter() + .any(|dep| dep.resource_type == MODEL_RESOURCE_TYPE && dep.name == model.name) + { + continue; + } + let (consumer_parts, passes_through) = consumer_part(context, model, consumer)?; + parts.push(consumer_parts); + if passes_through { + cascaded.push(consumer); + } + } + Ok((merge_parts(parts), cascaded)) +} + +fn consumer_part( + context: &ColumnRenameContext<'_>, + upstream: &ModelFacts, + consumer: &ModelFacts, +) -> Result<(RefactorParts, bool), RefactorError> { + let (body, compiled_sql, unmapped) = model_context(context, consumer); + let Some(body) = body else { + return Ok(( + RefactorParts { + manual: unmapped, + ..RefactorParts::default() + }, + false, + )); + }; + let analysis = analysis_sql(&chars(&compiled_sql), context.scan); + let facts = analyze_column( + &analysis, + &context.facts.dialect, + &context.columns, + &ColumnQuery { + column: &context.old, + target_tables: analysis.placeholders(REF_KIND, &upstream.name), + target_ctes: BTreeSet::new(), + output_scopes: BTreeSet::new(), + }, + )?; + let result = consumer_edits( + &facts, + &body, + context.names(), + (context.cascade, context.cascade), + ); + let file = context.file(body.path); + let header = match file { + Some(file) => consumer_column_header_edits( + &file.contents, + &file.text, + &upstream.name, + context.names(), + )?, + None => Vec::new(), + }; + let mut edits = result.edits; + edits.extend(header); + Ok(( + RefactorParts { + edits: path_edits(body.path, edits), + manual: result.manual, + ..RefactorParts::default() + }, + result.passes_through, + )) +} + +fn authored_body_parts( + context: &ColumnRenameContext<'_>, + model: &ModelFacts, +) -> Result { + let lowered_name = model.name.to_lowercase(); + let mut parts: Vec = Vec::new(); + for body in &context.bodies { + if body.body.to_lowercase().contains(&lowered_name) { + parts.push(authored_body(context, model, body)?); + } + } + Ok(merge_parts(parts)) +} + +fn authored_body( + context: &ColumnRenameContext<'_>, + model: &ModelFacts, + body: &AuthoredBody, +) -> Result { + let lowered = body.body.to_lowercase(); + let analysis = analysis_sql(&body.text, context.scan); + let targets = analysis.placeholders(REF_KIND, &model.name); + let fixture_scopes: BTreeSet = + if matches!(body.role, SqlFileRole::Test | SqlFileRole::Scenario) { + [ + format!("{EXPECTED_FIXTURE_PREFIX}{}", model.name), + format!("{REF_FIXTURE_PREFIX}{}", model.name), + ] + .into_iter() + .filter(|name| lowered.contains(&name.to_lowercase())) + .collect() + } else { + BTreeSet::new() + }; + if targets.is_empty() && fixture_scopes.is_empty() { + return Ok(RefactorParts::default()); + } + let facts = analyze_column( + &analysis, + &context.facts.dialect, + &context.columns, + &ColumnQuery { + column: &context.old, + target_tables: targets, + target_ctes: fixture_scopes + .iter() + .filter(|name| name.starts_with(REF_FIXTURE_PREFIX)) + .cloned() + .collect(), + output_scopes: fixture_scopes.clone(), + }, + )?; + let located = BodyContext { + path: &body.path, + contents: &body.contents, + mapping: BodyMapping::Authored(body.start), + fallback_offset: body.start, + }; + let outputs = output_edits(&facts, &located, &fixture_scopes, context.names()); + let consumers = consumer_edits(&facts, &located, context.names(), (false, true)); + let mut edits = outputs.edits; + edits.extend(consumers.edits); + let mut manual = outputs.manual; + manual.extend(consumers.manual); + Ok(RefactorParts { + edits: path_edits(&body.path, edits), + manual, + ..RefactorParts::default() + }) +} + +type ModelContext<'a> = (Option>, String, Vec); + +fn model_context<'a>( + context: &'a ColumnRenameContext<'_>, + model: &'a ModelFacts, +) -> ModelContext<'a> { + let Some(file) = context.file(&model.path) else { + return (None, String::new(), Vec::new()); + }; + let Some(body) = model_body(model, &file.text) else { + return ( + None, + String::new(), + vec![ManualLocation { + path: model.path.clone(), + line: None, + column: None, + reason: "the compiled SQL of this model cannot be mapped back to the file" + .to_owned(), + }], + ); + }; + let fallback_offset = body.body_start; + let compiled_sql = body.compiled_sql.clone(); + ( + Some(BodyContext { + path: &model.path, + contents: &file.text, + mapping: BodyMapping::Model(body), + fallback_offset, + }), + compiled_sql, + Vec::new(), + ) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_references.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_references.rs new file mode 100644 index 0000000000..38e6ad1a97 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/column_references.rs @@ -0,0 +1,444 @@ +//! Column-reference facts of one SQL body and their edits, as `column_references.py` derives. + +use std::collections::BTreeSet; + +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::refactoring::_helpers::edits::text_edits::{manual_at, text_edit}; +use crate::refactoring::_helpers::scanning::chars::slice; +use crate::refactoring::_helpers::scanning::sql_sites::{ + AnalysisSql, ModelBody, authored_offset, authored_span, +}; +use crate::refactoring::constants::{ + CTE_SCOPE_PREFIX, REF_KIND, ROOT_SCOPE, SEED_KIND, SOURCE_KIND, UNKNOWN_COLUMN_TYPE, +}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{EditKind, ManualLocation, RefactorFacts, TextEdit}; + +/// The known output columns of every model, source and seed, keyed by `(kind, name)`. +pub(crate) type ResourceColumns = Vec<((String, String), Vec)>; + +#[derive(Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ColumnFacts { + #[serde(default)] + pub(crate) parsed: bool, + #[serde(default)] + references: Vec, + #[serde(default)] + stars: Vec, + #[serde(default)] + joins: Vec, + #[serde(default)] + unresolved: Vec, + #[serde(default)] + outputs: Vec, + #[serde(default)] + star_outputs: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Projection { + alias: Option, + alias_start: Option, + alias_end: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ColumnReference { + start: usize, + end: usize, + name_start: usize, + name_end: usize, + scope: String, + projection: Option, +} + +#[derive(Debug, Deserialize)] +struct ColumnSite { + scope: String, + start: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct OutputColumn { + scope: String, + start: Option, + end: Option, + alias_start: Option, + alias_end: Option, +} + +/// Which column of which relations one analysis resolves. +pub(crate) struct ColumnQuery<'a> { + pub(crate) column: &'a str, + pub(crate) target_tables: BTreeSet, + pub(crate) target_ctes: BTreeSet, + pub(crate) output_scopes: BTreeSet, +} + +/// How one analysed SQL body maps onto its file. +pub(crate) enum BodyMapping { + /// A model's compiled SQL, mapped back through its expansion passes. + Model(ModelBody), + /// An authored body that starts at this file offset. + Authored(usize), +} + +/// Where one analysed SQL body lives, and how its offsets map onto the file. +pub(crate) struct BodyContext<'a> { + pub(crate) path: &'a str, + pub(crate) contents: &'a [char], + pub(crate) mapping: BodyMapping, + pub(crate) fallback_offset: usize, +} + +impl BodyContext<'_> { + fn map_span(&self, start: usize, end: usize) -> Option<(usize, usize)> { + match &self.mapping { + BodyMapping::Model(body) => authored_span(body, start, end), + BodyMapping::Authored(offset) => Some((offset + start, offset + end)), + } + } + + fn locate(&self, offset: usize) -> usize { + match &self.mapping { + BodyMapping::Model(body) => authored_offset(body, offset).0, + BodyMapping::Authored(start) => start + offset, + } + } + + fn manual(&self, offset: Option, reason: String) -> ManualLocation { + manual_at( + self.path, + self.contents, + Some(offset.unwrap_or(self.fallback_offset)), + reason, + ) + } + + fn rename_span(&self, span: (usize, usize), new: &str) -> TextEdit { + let authored = slice(self.contents, span.0, span.1); + text_edit( + self.contents, + span, + respell(&authored, new), + EditKind::Column, + ) + } +} + +/// Edits, manual locations, and root pass-throughs found in one SQL body. +#[derive(Default)] +pub(crate) struct BodyEdits { + pub(crate) edits: Vec, + pub(crate) manual: Vec, + pub(crate) passes_through: bool, +} + +impl BodyEdits { + fn extend(&mut self, other: Self) { + self.edits.extend(other.edits); + self.manual.extend(other.manual); + self.passes_through = self.passes_through || other.passes_through; + } + + fn manual(location: ManualLocation) -> Self { + Self { + manual: vec![location], + ..Self::default() + } + } +} + +fn set_columns( + mut columns: ResourceColumns, + key: (String, String), + names: Vec, +) -> ResourceColumns { + match columns.iter_mut().find(|(item, _)| *item == key) { + Some(entry) => entry.1 = names, + None => columns.push((key, names)), + } + columns +} + +/// The known output columns of every model, source, and seed. +pub(crate) fn resource_columns(facts: &RefactorFacts) -> ResourceColumns { + let mut columns: ResourceColumns = Vec::new(); + for model in &facts.models { + let mut names: Vec = model.inferred_columns.clone(); + for column in model.schema_columns.iter().flatten() { + if !names.contains(&column.name) { + names.push(column.name.clone()); + } + } + columns = set_columns(columns, (REF_KIND.to_owned(), model.name.clone()), names); + } + for source in &facts.sources { + columns = set_columns( + columns, + (SOURCE_KIND.to_owned(), source.name.clone()), + source.columns.clone(), + ); + } + for seed in &facts.seeds { + columns = set_columns( + columns, + (SEED_KIND.to_owned(), seed.name.clone()), + seed.columns.clone(), + ); + } + columns +} + +/// The columns of one resource, or none. +pub(crate) fn columns_of<'a>(columns: &'a ResourceColumns, kind: &str, name: &str) -> &'a [String] { + columns + .iter() + .find(|((item_kind, item_name), _)| item_kind == kind && item_name == name) + .map_or(&[], |(_, names)| names.as_slice()) +} + +/// Resolve every use of one column of the target relations in one SQL body. +pub(crate) fn analyze_column( + analysis: &AnalysisSql, + dialect: &str, + columns: &ResourceColumns, + query: &ColumnQuery<'_>, +) -> Result { + let mut tables: Vec = Vec::new(); + for ((kind, name), placeholders) in &analysis.tables { + let known: Vec = columns_of(columns, kind, name) + .iter() + .map(|column| json!({"name": column, "type": UNKNOWN_COLUMN_TYPE})) + .collect(); + for placeholder in placeholders { + tables.push(json!({"name": placeholder, "columns": known.clone()})); + } + } + let request = json!({ + "sql": analysis.sql, + "dialect": dialect, + "column": query.column, + "schema": {"strict": false, "tables": tables}, + "target_tables": query.target_tables, + "target_ctes": query.target_ctes, + "output_ctes": query.output_scopes, + }); + let response = + sqlbuild_analysis::column_references::main::analyze::analyze_json(&request.to_string()) + .map_err(RefactorError::value)?; + serde_json::from_str(&response).map_err(|error| RefactorError::value(error.to_string())) +} + +/// Rename every reference; keep consumer output names unless the root passes it on. +pub(crate) fn consumer_edits( + facts: &ColumnFacts, + context: &BodyContext<'_>, + names: (&str, &str), + stars: (bool, bool), +) -> BodyEdits { + let (cascade_root, root_stars_pass) = stars; + let (old, _) = names; + if !facts.parsed { + return BodyEdits::manual(context.manual(None, "SQL could not be analysed".to_owned())); + } + let mut result = BodyEdits::default(); + for reference in &facts.references { + result.extend(reference_edits(reference, context, names, cascade_root)); + } + for star in &facts.stars { + result.extend(star_edits(star, context, old, root_stars_pass)); + } + for site in &facts.joins { + result.extend(finding( + context, + site, + format!("USING or NATURAL join on {old}; rewrite it as an ON condition"), + )); + } + for site in &facts.unresolved { + result.extend(finding( + context, + site, + format!("cannot tell whether {old} here is the renamed column; qualify it"), + )); + } + result +} + +fn reference_edits( + reference: &ColumnReference, + context: &BodyContext<'_>, + names: (&str, &str), + cascade_root: bool, +) -> BodyEdits { + let (old, new) = names; + let Some(name_span) = context.map_span(reference.name_start, reference.name_end) else { + return BodyEdits::manual(context.manual( + Some(context.locate(reference.name_start)), + format!( + "column {old} is referenced in SQL a macro generates; update the macro call by hand" + ), + )); + }; + let authored_name = slice(context.contents, name_span.0, name_span.1); + let renamed = context.rename_span(name_span, new); + let Some(projection) = &reference.projection else { + return BodyEdits { + edits: vec![renamed], + ..BodyEdits::default() + }; + }; + let same_name_alias = projection + .alias + .as_ref() + .is_some_and(|alias| alias.to_lowercase() == old.to_lowercase()); + if reference.scope == ROOT_SCOPE + && cascade_root + && (projection.alias.is_none() || same_name_alias) + { + let alias_span = match ( + same_name_alias, + projection.alias_start, + projection.alias_end, + ) { + (true, Some(start), Some(end)) => context.map_span(start, end), + _ => None, + }; + let mut edits = vec![renamed]; + edits.extend(alias_span.map(|span| context.rename_span(span, new))); + return BodyEdits { + edits, + manual: Vec::new(), + passes_through: true, + }; + } + if projection.alias.is_some() { + return BodyEdits { + edits: vec![renamed], + ..BodyEdits::default() + }; + } + let Some(end_span) = context.map_span(reference.start, reference.end) else { + return BodyEdits::manual(context.manual( + Some(name_span.0), + format!("column {old} is projected inside a macro call"), + )); + }; + let alias = text_edit( + context.contents, + (end_span.1, end_span.1), + format!(" AS {authored_name}"), + EditKind::Column, + ) + .with_display(Some(String::new()), Some(format!("AS {authored_name}"))); + BodyEdits { + edits: vec![renamed, alias], + ..BodyEdits::default() + } +} + +fn star_edits( + star: &ColumnSite, + context: &BodyContext<'_>, + old: &str, + root_stars_pass: bool, +) -> BodyEdits { + if star.scope.starts_with(CTE_SCOPE_PREFIX) { + return BodyEdits::default(); + } + if star.scope == ROOT_SCOPE && root_stars_pass { + return BodyEdits { + passes_through: true, + ..BodyEdits::default() + }; + } + let reason = if star.scope == ROOT_SCOPE { + format!( + "SELECT * passes {old} through, so this model's output column would be renamed too; rerun with --cascade or list the columns" + ) + } else { + format!("SELECT * in a subquery passes {old} through; list the columns") + }; + finding(context, star, reason) +} + +fn finding(context: &BodyContext<'_>, site: &ColumnSite, reason: String) -> BodyEdits { + BodyEdits::manual(context.manual(site.start.map(|start| context.locate(start)), reason)) +} + +/// Rename the projection that names the column in the given output scopes. +pub(crate) fn output_edits( + facts: &ColumnFacts, + context: &BodyContext<'_>, + scopes: &BTreeSet, + names: (&str, &str), +) -> BodyEdits { + let (old, new) = names; + let mut result = BodyEdits::default(); + let mut found: BTreeSet<&str> = BTreeSet::new(); + for output in &facts.outputs { + found.insert(&output.scope); + if let (Some(alias_start), Some(alias_end)) = (output.alias_start, output.alias_end) { + match context.map_span(alias_start, alias_end) { + Some(span) => result.edits.push(context.rename_span(span, new)), + None => result.manual.push(context.manual( + Some(context.locate(alias_start)), + format!("column {old} is named inside a macro call"), + )), + } + continue; + } + let (Some(start), Some(end)) = (output.start, output.end) else { + continue; + }; + match context.map_span(start, end) { + Some(span) => result.edits.push( + text_edit( + context.contents, + (span.1, span.1), + format!(" AS {new}"), + EditKind::Column, + ) + .with_display(Some(String::new()), Some(format!("AS {new}"))), + ), + None => result.manual.push(context.manual( + Some(context.locate(start)), + format!("column {old} is produced inside a macro call"), + )), + } + } + for scope in scopes { + let role = scope_role(scope); + if !found.contains(role.as_str()) && facts.star_outputs.contains(&role) { + result.manual.push(context.manual( + None, + format!("{old} comes from SELECT * in {scope}; list the columns explicitly"), + )); + } + } + result +} + +fn scope_role(scope: &str) -> String { + if scope == ROOT_SCOPE { + scope.to_owned() + } else { + format!("{CTE_SCOPE_PREFIX}{scope}") + } +} + +fn respell(authored: &str, name: &str) -> String { + match authored.chars().next() { + Some('"') => format!("\"{name}\""), + Some('`') => format!("`{name}`"), + Some('[') => format!("[{name}]"), + _ => name.to_owned(), + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/declaration_moves.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/declaration_moves.rs new file mode 100644 index 0000000000..776e29a261 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/declaration_moves.rs @@ -0,0 +1,121 @@ +//! Declaration files a cross-folder model move takes along, as `model_planning.py` worked out. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use crate::refactoring::_helpers::files::paths::{join, parent}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{ + DeclarationMoves, DeclarationPlacement, ManualLocation, PlacedDeclaration, +}; +use crate::refactoring::types::DeclarationMoveHost; + +const UNSETTLED_REASON: &str = "declaration placement at the destination could not be worked out"; + +/// The declaration file moves of moving `model` from `source` to `destination`, and blockers. +pub(crate) fn declaration_moves( + project_dir: &Path, + model: &str, + paths: (&str, &str), + host: &DeclarationMoveHost<'_>, +) -> Result { + let (source, destination) = paths; + if parent(destination) == parent(source) { + return Ok(DeclarationMoves::default()); + } + let placement: DeclarationPlacement = host(model, destination)?; + let Some(relocated) = &placement.relocated else { + return Ok(unsettled(destination, &placement.diagnostics)); + }; + let mut destinations: BTreeMap = BTreeMap::new(); + let mut blocking: Vec = Vec::new(); + for record in relocated { + let Some(original) = placement + .declarations + .iter() + .find(|item| item.key == record.key) + else { + continue; + }; + if !join(project_dir, &original.path).is_file() { + blocking.push(blocker( + original, + format!( + "{} must move to {}, but it is not an authored project file", + original.label, record.path + ), + )); + continue; + } + let planned: &String = destinations + .entry(original.path.clone()) + .or_insert_with(|| record.path.clone()); + if *planned != record.path { + let reason = format!( + "{} holds declarations that must move to different folders ({planned}, {}); \ + split the file first", + original.path, record.path + ); + blocking.push(blocker(original, reason)); + } + } + let moving: BTreeSet<&str> = relocated.iter().map(|item| item.key.as_str()).collect(); + for staying in &placement.declarations { + if let Some(target) = destinations.get(&staying.path) + && !moving.contains(staying.key.as_str()) + { + blocking.push(blocker( + staying, + format!( + "{target} would take {} along, which must stay in {}; split the file first", + staying.label, staying.path + ), + )); + } + } + let targets: BTreeSet<&String> = destinations.values().collect(); + blocking.extend( + targets + .into_iter() + .filter(|path| join(project_dir, path).exists()) + .map(|path| ManualLocation { + path: path.clone(), + line: None, + column: None, + reason: "a declaration must move here, but the file already exists".to_owned(), + }), + ); + Ok(DeclarationMoves { + moves: destinations.into_iter().collect(), + blocking, + }) +} + +fn unsettled(destination: &str, diagnostics: &[String]) -> DeclarationMoves { + let reasons: Vec = if diagnostics.is_empty() { + vec![UNSETTLED_REASON.to_owned()] + } else { + diagnostics.to_vec() + }; + DeclarationMoves { + moves: Vec::new(), + blocking: reasons + .into_iter() + .map(|reason| ManualLocation { + path: destination.to_owned(), + line: None, + column: None, + reason, + }) + .collect(), + } +} + +fn blocker(record: &PlacedDeclaration, reason: String) -> ManualLocation { + ManualLocation { + path: record.path.clone(), + line: record.line, + column: record.column, + reason, + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs new file mode 100644 index 0000000000..0554329300 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod column_planning; +pub(crate) mod column_references; +pub(crate) mod declaration_moves; +pub(crate) mod model_planning; +pub(crate) mod model_references; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/model_planning.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/model_planning.rs new file mode 100644 index 0000000000..2a1bbc82e3 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/model_planning.rs @@ -0,0 +1,368 @@ +//! Resolve, block and migrate a model rename or move, as `model_planning.py` does. + +use std::path::Path; + +use crate::refactoring::_helpers::edits::header_edits::{header_tokens, is_value}; +use crate::refactoring::_helpers::edits::text_edits::{RefactorParts, manual_at}; +use crate::refactoring::_helpers::edits::yaml_edits::yaml_model_edits; +use crate::refactoring::_helpers::files::paths::{ + join, name, relative_posix, resolve, stem, suffix, with_name, +}; +use crate::refactoring::_helpers::files::project_files::{ + ProjectSqlFile, project_sql_files, yaml_files, +}; +use crate::refactoring::_helpers::planning::declaration_moves::declaration_moves; +use crate::refactoring::_helpers::planning::model_references::{ + macro_reference_locations, model_reference_edits, +}; +use crate::refactoring::_helpers::scanning::scan_context::ScanContext; +use crate::refactoring::constants::{ + CURRENT_DIRECTORY, HISTORY_MATERIALIZATIONS, MIGRATABLE_MATERIALIZATIONS, MIGRATE_FROM_KEY, + SQL_SUFFIX, +}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{ + ManualLocation, ModelFacts, RefactorFacts, RefactorOperation, RefactorRequest, TextEdit, +}; +use crate::refactoring::types::DeclarationMoveHost; + +/// The model a request names, and the request with its name and destination resolved. +pub(crate) struct ModelTarget<'a> { + pub(crate) model: &'a ModelFacts, + pub(crate) request: RefactorRequest, + pub(crate) source_path: String, +} + +impl ModelTarget<'_> { + pub(crate) fn destination(&self) -> String { + self.request + .destination + .clone() + .filter(|destination| !destination.is_empty()) + .unwrap_or_else(|| self.source_path.clone()) + } +} + +/// Whether a migration is needed, and why, or why none can be written. +pub(crate) struct ModelMigrationDecision { + pub(crate) needed: bool, + pub(crate) reason: String, + pub(crate) blocked: bool, +} + +/// The compiled model a target names. +pub(crate) fn find_model<'a>( + models: &'a [ModelFacts], + name: &str, +) -> Result<&'a ModelFacts, RefactorError> { + models + .iter() + .find(|model| model.name == name) + .ok_or_else(|| { + RefactorError::input( + "C951", + format!("no model named '{name}' in this project"), + Some("list models with sqb dag --json or sqb scope --browse models"), + ) + }) +} + +/// `IDENTIFIER_PATTERN.match`, whose `$` also matches before a trailing newline. +fn is_identifier(name: &str) -> bool { + let name = name.strip_suffix('\n').unwrap_or(name); + let mut characters = name.chars(); + characters + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == '_') + && characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +/// Refuse a new name that is not a plain SQL identifier. +pub(crate) fn validate_identifier(name: &str, noun: &str) -> Result<(), RefactorError> { + if is_identifier(name) { + return Ok(()); + } + Err(RefactorError::input( + "C952", + format!("'{name}' is not a valid {noun} name"), + Some("use letters, digits, and underscores, starting with a letter or underscore"), + )) +} + +/// Resolve the model, its new name, and its destination file. +pub(crate) fn model_target<'a>( + facts: &'a RefactorFacts, + request: &RefactorRequest, +) -> Result, RefactorError> { + let model = find_model(&facts.models, &request.model_name)?; + let source_path = model.path.clone(); + let is_move = request.operation == RefactorOperation::MoveModel; + let destination = if is_move { + resolve_destination( + Path::new(&facts.project_dir), + request.destination.as_deref().unwrap_or_default(), + name(&source_path), + )? + } else { + with_name(&source_path, &format!("{}{SQL_SUFFIX}", request.new_name)) + }; + let new = if is_move { + stem(&destination).to_owned() + } else { + request.new_name.clone() + }; + validate_identifier(&new, "model")?; + if new == model.name && destination == source_path { + return Err(RefactorError::input( + "C953", + format!("model:{} already has that name and location", model.name), + None, + )); + } + Ok(ModelTarget { + model, + request: RefactorRequest { + new_name: new, + destination: Some(destination), + ..request.clone() + }, + source_path, + }) +} + +fn resolve_destination( + project_dir: &Path, + raw: &str, + file_name: &str, +) -> Result { + let path = Path::new(raw); + let mut absolute = if path.is_absolute() { + path.to_path_buf() + } else { + project_dir.join(path) + }; + if raw.ends_with('/') || absolute.is_dir() { + absolute = absolute.join(file_name); + } + let resolved = resolve(&absolute); + let Some(relative) = relative_posix(&resolved, project_dir) else { + return Err(RefactorError::input( + "C955", + format!("destination '{raw}' is outside the project"), + None, + )); + }; + if suffix(name(&relative)) != SQL_SUFFIX || relative == CURRENT_DIRECTORY { + return Err(RefactorError::input( + "C955", + format!("destination '{raw}' must be a .sql file or a folder ending in /"), + None, + )); + } + Ok(relative) +} + +/// Every edit, manual location, and blocker of one model rename or move. +pub(crate) fn model_parts( + facts: &RefactorFacts, + target: &ModelTarget<'_>, + context: &ScanContext, + host: &DeclarationMoveHost<'_>, +) -> Result { + let files = project_sql_files(facts); + let old = target.model.name.as_str(); + let new = target.request.new_name.as_str(); + let declaration = declaration_moves( + Path::new(&facts.project_dir), + &target.model.name, + (&target.source_path, &target.destination()), + host, + )?; + let mut edits: Vec<(String, TextEdit)> = Vec::new(); + if new != old { + edits.extend(model_reference_edits(&files, (old, new), context)?); + edits.extend(yaml_model_edits(&yaml_files(facts), old, new)?); + } + let mut manual = macro_reference_locations(&facts.models, &files, old, context); + manual.extend(facts.python_locations.iter().cloned()); + let mut blocking = collisions(facts, target); + blocking.extend(declaration.blocking); + blocking.extend(pending_migration(&files, target)?); + Ok(RefactorParts { + edits, + manual, + blocking, + moves: declaration.moves, + ..RefactorParts::default() + }) +} + +fn collisions(facts: &RefactorFacts, target: &ModelTarget<'_>) -> Vec { + let new = &target.request.new_name; + let mut found: Vec = facts + .models + .iter() + .filter(|other| { + other.name.to_lowercase() == new.to_lowercase() && other.name != target.model.name + }) + .map(|other| ManualLocation { + path: other.path.clone(), + line: None, + column: None, + reason: format!("model:{new} already exists"), + }) + .collect(); + let destination = target.destination(); + if destination != target.source_path + && join(Path::new(&facts.project_dir), &destination).exists() + { + found.push(ManualLocation { + path: destination, + line: None, + column: None, + reason: "destination file already exists".to_owned(), + }); + } + found +} + +fn pending_migration( + files: &[ProjectSqlFile], + target: &ModelTarget<'_>, +) -> Result, RefactorError> { + if !target.model.declares_migrate_from { + return Ok(Vec::new()); + } + let (contents, text) = files + .iter() + .find(|file| file.path == target.source_path) + .map(|file| (file.contents.clone(), file.text.clone())) + .unwrap_or_default(); + let offset = header_tokens(&contents, &text)? + .unwrap_or_default() + .iter() + .find(|token| token.depth == 0 && is_value(token, MIGRATE_FROM_KEY)) + .map(|token| token.start); + Ok(vec![manual_at( + &target.source_path, + &text, + offset, + format!( + "model:{} still declares migrate_from; build it on every target and remove migrate_from before renaming it again", + target.model.name + ), + )]) +} + +fn python_none(value: Option<&str>) -> String { + value.map_or_else(|| "None".to_owned(), str::to_owned) +} + +/// Declare `migrate_from` whenever the relation moves, unless migrations cannot follow it. +pub(crate) fn decide_model_migration( + before: &[ModelFacts], + after: Option<&[ModelFacts]>, + names: (&str, &str), +) -> ModelMigrationDecision { + let (old, new) = names; + let decision = |needed: bool, reason: String| ModelMigrationDecision { + needed, + reason, + blocked: false, + }; + let Some(old_model) = before.iter().find(|model| model.name == old) else { + return decision(false, "model not compiled".to_owned()); + }; + let new_model = after.and_then(|models| models.iter().find(|model| model.name == new)); + let materialized = new_model.unwrap_or(old_model).materialized.as_deref(); + if !materialized.is_some_and(|value| MIGRATABLE_MATERIALIZATIONS.contains(&value)) { + return decision( + false, + format!( + "'{}' models keep no warehouse data", + python_none(materialized) + ), + ); + } + let (Some(after), Some(new_model)) = (after, new_model) else { + return decision( + true, + "keeps the relation's history; the destination could not be checked because the edited project does not compile" + .to_owned(), + ); + }; + if location_key(old_model) == location_key(new_model) { + return decision(false, "relation name is unchanged".to_owned()); + } + blocked_move(after, old_model, new_model).unwrap_or_else(|| { + decision( + true, + "keeps the relation's history and its old name working".to_owned(), + ) + }) +} + +fn blocked_move( + after: &[ModelFacts], + old_model: &ModelFacts, + new_model: &ModelFacts, +) -> Option { + let lowered = |value: &Option| value.as_deref().unwrap_or_default().to_lowercase(); + let old_database = lowered(&old_model.destination.database); + let new_database = lowered(&new_model.destination.database); + if !old_database.is_empty() && !new_database.is_empty() && old_database != new_database { + return Some(ModelMigrationDecision { + needed: false, + blocked: true, + reason: format!( + "moves from database {} to {}; migrations cannot cross databases", + python_none(old_model.destination.database.as_deref()), + python_none(new_model.destination.database.as_deref()) + ), + }); + } + let old_schema = lowered(&old_model.destination.schema); + if after + .iter() + .any(|model| lowered(&model.destination.schema) == old_schema) + { + return None; + } + Some(ModelMigrationDecision { + needed: false, + blocked: true, + reason: format!( + "no model is left in schema {}, so migrate_from {} cannot find the old relation; add a schema-qualified migrate_from for each target", + python_none(old_model.destination.schema.as_deref()), + old_model.name + ), + }) +} + +/// Whether renaming a column of this model must declare `migrate_from`. +pub(crate) fn needs_column_migration(model: &ModelFacts) -> bool { + let materialized = model.materialized.as_deref(); + if materialized.is_some_and(|value| HISTORY_MATERIALIZATIONS.contains(&value)) { + return true; + } + materialized.is_some_and(|value| MIGRATABLE_MATERIALIZATIONS.contains(&value)) + && model.migrate_from_set +} + +fn location_key(model: &ModelFacts) -> (String, String, String) { + ( + model + .destination + .database + .as_deref() + .unwrap_or_default() + .to_lowercase(), + model + .destination + .schema + .as_deref() + .unwrap_or_default() + .to_lowercase(), + model.destination.name.to_lowercase(), + ) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/model_references.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/model_references.rs new file mode 100644 index 0000000000..973c6fa8eb --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/planning/model_references.rs @@ -0,0 +1,174 @@ +//! Authored model references and unrewritable ones, as `model_references.py` finds them. + +use crate::refactoring::_helpers::edits::header_edits::{ + model_name_header_edits, schema_model_name_edits, +}; +use crate::refactoring::_helpers::edits::text_edits::{ + identifier_sites, manual_at, path_edits, text_edit, +}; +use crate::refactoring::_helpers::files::project_files::ProjectSqlFile; +use crate::refactoring::_helpers::scanning::chars::{chars, slice}; +use crate::refactoring::_helpers::scanning::scan_context::ScanContext; +use crate::refactoring::_helpers::scanning::sql_sites::{ + ResourceSite, authored_offset, model_body, resource_sites, +}; +use crate::refactoring::constants::{ + EXPECTED_FIXTURE_PREFIX, MODEL_RESOURCE_TYPE, REF_FIXTURE_PREFIX, REF_KIND, +}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{EditKind, ManualLocation, ModelFacts, SqlFileRole, TextEdit}; + +fn ref_sites(text: &[char], name: &str, context: &ScanContext) -> Vec { + resource_sites(text, context) + .into_iter() + .filter(|site| site.kind == REF_KIND && site.name == name) + .collect() +} + +/// Rewrite `__ref`, fixture CTE names, and header references to a renamed model. +pub(crate) fn model_reference_edits( + files: &[ProjectSqlFile], + names: (&str, &str), + context: &ScanContext, +) -> Result, RefactorError> { + let (old, new) = names; + let mut edits: Vec<(String, TextEdit)> = Vec::new(); + for item in files { + edits.extend(path_edits(&item.path, ref_edits(item, names, context))); + if matches!(item.role, SqlFileRole::Test | SqlFileRole::Scenario) { + edits.extend(path_edits(&item.path, fixture_edits(item, names))); + } + if item.role == SqlFileRole::Model { + edits.extend(path_edits( + &item.path, + model_name_header_edits(&item.contents, &item.text, old, new)?, + )); + } + if item.role == SqlFileRole::Schema { + edits.extend(path_edits( + &item.path, + schema_model_name_edits(&item.contents, &item.text, names, context.python)?, + )); + } + } + Ok(edits) +} + +fn ref_edits(item: &ProjectSqlFile, names: (&str, &str), context: &ScanContext) -> Vec { + let (old, new) = names; + let text = &item.text; + ref_sites(text, old, context) + .into_iter() + .map(|site| { + let after = format!( + "{}{new}{}", + slice(text, site.start, site.name_start), + slice(text, site.name_end, site.end) + ); + text_edit( + text, + (site.name_start, site.name_end), + new.to_owned(), + EditKind::Reference, + ) + .with_display(Some(slice(text, site.start, site.end)), Some(after)) + }) + .collect() +} + +fn fixture_edits(item: &ProjectSqlFile, names: (&str, &str)) -> Vec { + let (old, new) = names; + let fixture_names: [(String, String); 2] = [ + ( + format!("{REF_FIXTURE_PREFIX}{old}"), + format!("{REF_FIXTURE_PREFIX}{new}"), + ), + ( + format!("{EXPECTED_FIXTURE_PREFIX}{old}"), + format!("{EXPECTED_FIXTURE_PREFIX}{new}"), + ), + ]; + let searched: Vec = fixture_names.iter().map(|(name, _)| name.clone()).collect(); + identifier_sites(&item.text, &searched) + .into_iter() + .filter_map(|(start, end, name)| { + let replacement = fixture_replacement(&fixture_names, &name)?; + Some(text_edit( + &item.text, + (start, end), + replacement, + EditKind::Fixture, + )) + }) + .collect() +} + +fn fixture_replacement(fixture_names: &[(String, String)], name: &str) -> Option { + fixture_names + .iter() + .find(|(item, _)| item == name) + .map(|(_, replacement)| replacement.clone()) +} + +fn depends_on(model: &ModelFacts, name: &str) -> bool { + model + .deps + .iter() + .any(|dep| dep.resource_type == MODEL_RESOURCE_TYPE && dep.name == name) +} + +/// `__ref` calls to the model that macros produce, which no edit can reach. +pub(crate) fn macro_reference_locations( + models: &[ModelFacts], + files: &[ProjectSqlFile], + old: &str, + context: &ScanContext, +) -> Vec { + let mut locations: Vec = Vec::new(); + for model in models { + let Some(file) = files.iter().find(|file| file.path == model.path) else { + continue; + }; + if !depends_on(model, old) { + continue; + } + locations.extend(generated_references(model, &file.text, old, context)); + } + locations +} + +fn generated_references( + model: &ModelFacts, + text: &[char], + old: &str, + context: &ScanContext, +) -> Vec { + let compiled_sites = ref_sites(&chars(&model.query_sql), old, context); + let authored_sites = ref_sites(&chars(&model.authored_query_sql), old, context); + if compiled_sites.len() <= authored_sites.len() { + return Vec::new(); + } + let Some(body) = model_body(model, text) else { + return vec![manual_at( + &model.path, + text, + None, + format!("reference to {old} produced by a macro"), + )]; + }; + compiled_sites + .iter() + .map(|site| authored_offset(&body, site.start)) + .filter(|(_, generated)| *generated) + .map(|(offset, _)| { + manual_at( + &model.path, + text, + Some(offset), + format!( + "reference to {old} produced by a macro; pass the model to the macro as an argument" + ), + ) + }) + .collect() +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/chars.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/chars.rs new file mode 100644 index 0000000000..6feb2d0c7d --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/chars.rs @@ -0,0 +1,67 @@ +//! Code-point text operations with Python `str` semantics. + +/// The code points of a text, so offsets match Python string offsets. +pub(crate) fn chars(text: &str) -> Vec { + text.chars().collect() +} + +/// `text[start:end]` with Python's clamping. +pub(crate) fn slice(text: &[char], start: usize, end: usize) -> String { + let end = end.min(text.len()); + let start = start.min(end); + text[start..end].iter().collect() +} + +/// `text.startswith(needle, at)`. +pub(crate) fn starts_with(text: &[char], at: usize, needle: &str) -> bool { + (at..) + .zip(needle.chars()) + .all(|(index, character)| text.get(index) == Some(&character)) +} + +/// `text.find(needle, from)`. +pub(crate) fn find(text: &[char], needle: &str, from: usize) -> Option { + let length = needle.chars().count(); + if length == 0 { + return (from <= text.len()).then_some(from); + } + (from..=text.len().saturating_sub(length)).find(|index| starts_with(text, *index, needle)) +} + +/// `text.rfind(needle, start, end)` for a non-empty needle. +pub(crate) fn rfind(text: &[char], needle: &str, start: usize, end: usize) -> Option { + let length = needle.chars().count(); + let end = end.min(text.len()); + if end < start + length { + return None; + } + (start..=end - length) + .rev() + .find(|index| starts_with(text, *index, needle)) +} + +/// `text.count(needle, 0, end)` for a one-character needle. +pub(crate) fn count_before(text: &[char], needle: char, end: usize) -> usize { + text[..end.min(text.len())] + .iter() + .filter(|character| **character == needle) + .count() +} + +/// Whether two characters match under `re.IGNORECASE`. +pub(crate) fn same_ignoring_case(left: char, right: char) -> bool { + left == right || left.to_lowercase().eq(right.to_lowercase()) +} + +/// Whether `needle` occurs at `at`, ignoring case per character as `re.IGNORECASE` does. +pub(crate) fn starts_with_ignoring_case(text: &[char], at: usize, needle: &[char]) -> bool { + needle.iter().enumerate().all(|(offset, character)| { + text.get(at + offset) + .is_some_and(|found| same_ignoring_case(*found, *character)) + }) +} + +/// `[A-Za-z0-9_$]`, the identifier characters of refactoring patterns. +pub(crate) fn is_identifier_character(character: char) -> bool { + character.is_ascii_alphanumeric() || matches!(character, '_' | '$') +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/interpolation.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/interpolation.rs new file mode 100644 index 0000000000..16b16ff9cb --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/interpolation.rs @@ -0,0 +1,338 @@ +//! An exact port of `scan_interpolation_sites` with the lint lexical policy. + +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; +use sqlbuild_core::text::main::is_python_alpha::is_python_alpha; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::models::PythonText; + +use crate::refactoring::_helpers::scanning::chars::{find, slice, starts_with}; + +/// The reference functions the lint scan pattern matches, in `sorted()` order. +const REFERENCE_FUNCTIONS: [&str; 6] = [ + "__dbt_ref", + "__ref", + "__seed", + "__source", + "__table_fn", + "__udf", +]; +/// Every SQLBuild function name `_interpolation_site_end` accepts. +const FUNCTION_NAMES: [&str; 9] = [ + "__cursor_start", + "__cursor_end", + "__empty_fixture", + "__dbt_ref", + "__table_fn", + "__source", + "__seed", + "__udf", + "__ref", +]; +const INTERPOLATION_NAME_EXTRA: [char; 3] = ['_', ':', '.']; + +/// One interpolation site in original text offsets. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct InterpolationSite { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) text: String, +} + +/// Every interpolation site of `body`, as `scan_interpolation_sites(body=..., dialect=...)`. +pub(crate) fn interpolation_sites( + body: &[char], + backtick_identifiers: bool, + python: PythonText, +) -> Vec { + let scanner = Scanner { + body, + backtick_identifiers, + python, + }; + let mut sites: Vec = Vec::new(); + let mut copied_to = 0; + let mut index = 0; + while index < body.len() { + match scanner.pattern_match(index) { + Some(PatternMatch::NonCode(end)) => index = end, + Some(PatternMatch::Site(end)) => { + if index >= copied_to + && let Some(site_end) = scanner.site_end(index) + { + sites.push(InterpolationSite { + start: index, + end: site_end, + text: slice(body, index, site_end), + }); + copied_to = site_end; + } + index = end; + } + None => index += 1, + } + } + sites +} + +enum PatternMatch { + NonCode(usize), + Site(usize), +} + +struct Scanner<'a> { + body: &'a [char], + backtick_identifiers: bool, + python: PythonText, +} + +impl Scanner<'_> { + fn at(&self, index: usize) -> Option { + self.body.get(index).copied() + } + + /// The combined scan pattern at `index`: non-code alternatives first, then sites. + fn pattern_match(&self, index: usize) -> Option { + let body = self.body; + match self.at(index)? { + '-' if self.at(index + 1) == Some('-') => { + let end = (index + 2..body.len()) + .find(|position| body[*position] == '\n') + .map_or(body.len(), |position| position + 1); + return Some(PatternMatch::NonCode(end)); + } + '/' if self.at(index + 1) == Some('*') => { + let end = find(body, "*/", index + 2).map_or(body.len(), |position| position + 2); + return Some(PatternMatch::NonCode(end)); + } + quote @ ('\'' | '"') => { + if let Some(end) = self.backslash_quoted_end(index, quote) { + return Some(PatternMatch::NonCode(end)); + } + } + '$' => { + if let Some(end) = self.dollar_quote_end(index) { + return Some(PatternMatch::NonCode(end)); + } + } + '`' if self.backtick_identifiers => { + let mut position = index + 1; + while position < body.len() { + if body[position] == '`' { + if self.at(position + 1) == Some('`') { + position += 2; + continue; + } + return Some(PatternMatch::NonCode(position + 1)); + } + position += 1; + } + return Some(PatternMatch::NonCode(body.len())); + } + _ => {} + } + self.site_pattern_end(index).map(PatternMatch::Site) + } + + /// Python's quoted-string regex, backtracking to the last doubled quote after `\` + newline. + fn backslash_quoted_end(&self, index: usize, quote: char) -> Option { + let body = self.body; + let mut position = index + 1; + let mut last_doubled: Option = None; + loop { + match self.at(position) { + None => return Some(body.len()), + Some('\\') => match self.at(position + 1) { + Some(next) if next != '\n' => position += 2, + _ => return last_doubled.map(|doubled| doubled + 1), + }, + Some(character) if character == quote => { + if self.at(position + 1) == Some(quote) { + last_doubled = Some(position); + position += 2; + continue; + } + return Some(position + 1); + } + Some(_) => position += 1, + } + } + } + + /// `(? Option { + if index > 0 { + let previous = self.body[index - 1]; + if previous.is_ascii_alphanumeric() + || matches!(previous, '_' | '$') + || !previous.is_ascii() + { + return None; + } + } + let mut position = index + 1; + if self + .at(position) + .is_some_and(|character| character.is_ascii_alphabetic() || character == '_') + { + position += 1; + while self + .at(position) + .is_some_and(|character| character.is_ascii_alphanumeric() || character == '_') + { + position += 1; + } + } + if self.at(position) != Some('$') { + return None; + } + let delimiter = slice(self.body, index, position + 1); + Some( + find(self.body, &delimiter, position + 1) + .map_or(self.body.len(), |close| close + delimiter.chars().count()), + ) + } + + /// `(?P@@|\$\{|@|(?:functions)\()` at `index`. + fn site_pattern_end(&self, index: usize) -> Option { + let body = self.body; + if starts_with(body, index, "@@") { + return Some(index + 2); + } + if starts_with(body, index, "${") { + return Some(index + 2); + } + if self.at(index) == Some('@') { + return Some(index + 1); + } + REFERENCE_FUNCTIONS.iter().find_map(|name| { + let length = name.chars().count(); + (starts_with(body, index, name) && self.at(index + length) == Some('(')) + .then_some(index + length + 1) + }) + } + + /// `_interpolation_site_end`. + fn site_end(&self, start: usize) -> Option { + let character = self.at(start)?; + if character == '@' { + if starts_with(self.body, start, "@@") { + return Some(self.interpolation_name_end(start + 2)); + } + return self.macro_site_end(start); + } + if character == '$' { + return find(self.body, "}", start + 2).map(|end| end + 1); + } + if character == '_' { + let name = slice(self.body, start, self.identifier_end(start)).to_lowercase(); + if FUNCTION_NAMES.contains(&name.as_str()) { + return self.function_site_end(start); + } + } + None + } + + fn function_site_end(&self, start: usize) -> Option { + let mut name_end = self.identifier_end(start); + while self.at(name_end).is_some_and(is_python_space) { + name_end += 1; + } + if self.at(name_end) != Some('(') { + return None; + } + self.matching_paren_end(name_end) + } + + fn macro_site_end(&self, start: usize) -> Option { + let name_start = start + 1; + let first = self.at(name_start)?; + if first == '\'' { + return find(self.body, "'", name_start + 1).map(|closing| closing + 1); + } + if !(is_python_alpha(self.python, first) || first == '_') { + return None; + } + let name_end = self.identifier_end(name_start); + if self.at(name_end) != Some('(') { + return Some(name_end); + } + self.matching_paren_end(name_end) + } + + fn matching_paren_end(&self, opening: usize) -> Option { + let body = self.body; + let mut depth: i64 = 0; + let mut index = opening; + let mut quote: Option = None; + while index < body.len() { + if quote.is_none() && starts_with(body, index, "--") { + index = find(body, "\n", index + 2).map_or(body.len(), |newline| newline + 1); + continue; + } + if quote.is_none() && starts_with(body, index, "/*") { + index = find(body, "*/", index + 2).map_or(body.len(), |end| end + 2); + continue; + } + if quote.is_none() + && body[index] == '$' + && let Some(end) = self.dollar_quote_end(index) + { + index = end; + continue; + } + let character = body[index]; + if let Some(open) = quote { + if character == '\\' && matches!(open, '\'' | '"') && index + 1 < body.len() { + index += 2; + continue; + } + if character == open { + if self.at(index + 1) == Some(open) { + index += 2; + continue; + } + quote = None; + } + index += 1; + continue; + } + if matches!(character, '\'' | '"') || (self.backtick_identifiers && character == '`') { + quote = Some(character); + index += 1; + continue; + } + if character == '(' { + depth += 1; + } + if character == ')' { + depth -= 1; + if depth == 0 { + return Some(index + 1); + } + } + index += 1; + } + None + } + + fn interpolation_name_end(&self, start: usize) -> usize { + let mut index = start; + while self.at(index).is_some_and(|character| { + is_python_alnum(self.python, character) || INTERPOLATION_NAME_EXTRA.contains(&character) + }) { + index += 1; + } + index + } + + fn identifier_end(&self, start: usize) -> usize { + let mut index = start; + while self + .at(index) + .is_some_and(|character| is_python_alnum(self.python, character) || character == '_') + { + index += 1; + } + index + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/mod.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/mod.rs new file mode 100644 index 0000000000..b71538eaee --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod chars; +pub(crate) mod interpolation; +pub(crate) mod scan_context; +pub(crate) mod sql_sites; diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/scan_context.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/scan_context.rs new file mode 100644 index 0000000000..6d3ff62037 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/scan_context.rs @@ -0,0 +1,32 @@ +//! The lexical settings every SQL scan of one refactoring shares. + +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; +use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; + +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::RefactorFacts; + +/// The dialect's quote policy and the Python string semantics of the host. +#[derive(Clone, Copy, Debug)] +pub(crate) struct ScanContext { + pub(crate) backtick_identifiers: bool, + pub(crate) python: PythonText, +} + +impl ScanContext { + /// The scan settings for a project's facts, or a deferral for an unknown Python. + pub(crate) fn for_facts(facts: &RefactorFacts) -> Result { + let python = + python_text(facts.python_version, &facts.unicode_version).ok_or_else(|| { + RefactorError::internal(format!( + "native refactoring does not support Python {}.{} with Unicode {}", + facts.python_version.0, facts.python_version.1, facts.unicode_version + )) + })?; + Ok(Self { + backtick_identifiers: quote_policy(&facts.dialect).backtick_identifiers, + python, + }) + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs new file mode 100644 index 0000000000..3d35cfd26c --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs @@ -0,0 +1,296 @@ +//! Call sites, analysis SQL and authored offsets, as `sql_sites.py` finds them. + +use std::collections::BTreeSet; + +use sqlbuild_core::text::main::is_python_space::is_python_space; + +use crate::refactoring::_helpers::scanning::chars::{chars, rfind, slice, starts_with}; +use crate::refactoring::_helpers::scanning::interpolation::interpolation_sites; +use crate::refactoring::_helpers::scanning::scan_context::ScanContext; +use crate::refactoring::constants::{GENERIC_PLACEHOLDER_KIND, PLACEHOLDER_PAD, REF_FUNCTION}; +use crate::refactoring::models::{ExpansionSpan, ModelFacts}; + +/// One `__ref`, `__source`, or `__seed` call written in authored SQL. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ResourceSite { + pub(crate) kind: String, + pub(crate) name: String, + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) name_start: usize, + pub(crate) name_end: usize, +} + +/// SQL whose interpolation sites were replaced by same-length identifiers. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct AnalysisSql { + pub(crate) sql: String, + /// Each resource call `(kind, name)` with its placeholders, in first appearance order. + pub(crate) tables: Vec<((String, String), BTreeSet)>, +} + +impl AnalysisSql { + /// The placeholders of one resource, or none. + pub(crate) fn placeholders(&self, kind: &str, name: &str) -> BTreeSet { + self.tables + .iter() + .find(|((item_kind, item_name), _)| item_kind == kind && item_name == name) + .map(|(_, placeholders)| placeholders.clone()) + .unwrap_or_default() + } +} + +/// A model's compiled query and how it maps onto the authored file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ModelBody { + pub(crate) body_start: usize, + pub(crate) compiled_sql: String, + pub(crate) passes: Vec>, +} + +/// `RESOURCE_CALL_PATTERN` on a site's text: `(kind, name, name_start, name_end)`. +fn resource_call(text: &[char]) -> Option<(String, String, usize, usize)> { + if !starts_with(text, 0, "__") { + return None; + } + let kind = ["ref", "source", "seed"] + .into_iter() + .find(|kind| starts_with(text, 2, kind))?; + let mut index = 2 + kind.len(); + index = skip_space(text, index); + if text.get(index) != Some(&'(') { + return None; + } + index = skip_space(text, index + 1); + let quote = *text + .get(index) + .filter(|character| matches!(character, '\'' | '"'))?; + let name_start = index + 1; + let mut name_end = name_start; + while text + .get(name_end) + .is_some_and(|character| !matches!(character, '\'' | '"')) + { + name_end += 1; + } + if name_end == name_start || text.get(name_end) != Some("e) { + return None; + } + index = skip_space(text, name_end + 1); + if text.get(index) != Some(&')') { + return None; + } + let rest = &text[index + 1..]; + if !(rest.is_empty() || rest == ['\n']) { + return None; + } + Some(( + kind.to_owned(), + slice(text, name_start, name_end), + name_start, + name_end, + )) +} + +fn skip_space(text: &[char], mut index: usize) -> usize { + while text.get(index).copied().is_some_and(is_python_space) { + index += 1; + } + index +} + +/// Resource calls outside comments and strings, with their name spans. +pub(crate) fn resource_sites(text: &[char], context: &ScanContext) -> Vec { + interpolation_sites(text, context.backtick_identifiers, context.python) + .into_iter() + .filter_map(|site| { + let (kind, name, name_start, name_end) = resource_call(&chars(&site.text))?; + Some(ResourceSite { + kind, + name, + start: site.start, + end: site.end, + name_start: site.start + name_start, + name_end: site.start + name_end, + }) + }) + .collect() +} + +/// `EMBEDDED_REF_PATTERN` name spans of `__ref` calls inside a quoted string's raw text. +pub(crate) fn embedded_ref_spans(text: &[char], name: &str) -> Vec<(usize, usize)> { + let mut spans: Vec<(usize, usize)> = Vec::new(); + let mut index = 0; + while index < text.len() { + match embedded_ref_at(text, index) { + Some((name_start, name_end, end)) => { + if slice(text, name_start, name_end) == name { + spans.push((name_start, name_end)); + } + index = end; + } + None => index += 1, + } + } + spans +} + +/// `__ref\s*\(\s*\\?['"](?P[^'"\\]+)\\?['"]\s*\)` at `index`: name span and match end. +fn embedded_ref_at(text: &[char], index: usize) -> Option<(usize, usize, usize)> { + if !starts_with(text, index, REF_FUNCTION) { + return None; + } + let mut position = skip_space(text, index + REF_FUNCTION.len()); + if text.get(position) != Some(&'(') { + return None; + } + position = skip_space(text, position + 1); + if text.get(position) == Some(&'\\') { + position += 1; + } + if !text + .get(position) + .is_some_and(|character| matches!(character, '\'' | '"')) + { + return None; + } + let name_start = position + 1; + let mut name_end = name_start; + while text + .get(name_end) + .is_some_and(|character| !matches!(character, '\'' | '"' | '\\')) + { + name_end += 1; + } + if name_end == name_start { + return None; + } + position = name_end; + if text.get(position) == Some(&'\\') { + position += 1; + } + if !text + .get(position) + .is_some_and(|character| matches!(character, '\'' | '"')) + { + return None; + } + position = skip_space(text, position + 1); + if text.get(position) != Some(&')') { + return None; + } + Some((name_start, name_end, position + 1)) +} + +/// Replace every interpolation site with an identifier of the same length. +pub(crate) fn analysis_sql(text: &[char], context: &ScanContext) -> AnalysisSql { + let mut sql = String::new(); + let mut tables: Vec<((String, String), BTreeSet)> = Vec::new(); + let mut copied_to = 0; + for (index, site) in interpolation_sites(text, context.backtick_identifiers, context.python) + .into_iter() + .enumerate() + { + let call = resource_call(&chars(&site.text)); + let kind: char = call + .as_ref() + .and_then(|(kind, ..)| kind.chars().next()) + .unwrap_or(GENERIC_PLACEHOLDER_KIND); + let placeholder = placeholder(kind, index, site.end - site.start); + if let Some((kind, name, ..)) = call { + let key = (kind, name); + match tables.iter_mut().find(|(item, _)| *item == key) { + Some((_, placeholders)) => { + placeholders.insert(placeholder.clone()); + } + None => tables.push((key, BTreeSet::from([placeholder.clone()]))), + } + } + sql.push_str(&slice(text, copied_to, site.start)); + sql.push_str(&placeholder); + copied_to = site.end; + } + sql.push_str(&slice(text, copied_to, text.len())); + AnalysisSql { sql, tables } +} + +fn placeholder(kind: char, index: usize, length: usize) -> String { + let base = format!("_q{kind}{index}"); + let base_length = base.chars().count(); + if base_length > length { + let mut padded: String = + std::iter::repeat_n(PLACEHOLDER_PAD, length.saturating_sub(1)).collect(); + padded.push(kind); + return padded; + } + let mut padded = base; + padded.extend(std::iter::repeat_n(PLACEHOLDER_PAD, length - base_length)); + padded +} + +/// The mappable compiled body of a model, or `None` when it cannot be mapped. +pub(crate) fn model_body(model: &ModelFacts, contents: &[char]) -> Option { + if model.authored_query_sql.is_empty() { + return None; + } + let body_start = rfind(contents, &model.authored_query_sql, 0, contents.len())?; + let passes = match &model.expansion { + Some(expansion) => { + if expansion.expanded_sql != model.query_sql { + return None; + } + expansion.passes.clone() + } + None if model.query_sql != model.authored_query_sql => return None, + None => Vec::new(), + }; + Some(ModelBody { + body_start, + compiled_sql: model.query_sql.clone(), + passes, + }) +} + +/// Resolve one rendered offset through one expansion pass (`map_output_offset`). +fn map_output_offset(offset: usize, spans: &[ExpansionSpan]) -> (usize, bool) { + let mut mapped = offset as i64; + for span in spans { + if offset < span.output_start { + break; + } + if offset < span.output_end { + return (span.source_start, true); + } + mapped += (span.source_end as i64 - span.source_start as i64) + - (span.output_end as i64 - span.output_start as i64); + } + (usize::try_from(mapped).unwrap_or(0), false) +} + +/// The file offset of a compiled offset and whether a macro generated it. +pub(crate) fn authored_offset(body: &ModelBody, offset: usize) -> (usize, bool) { + let mut current = offset; + let mut generated = false; + for spans in body.passes.iter().rev() { + let (resolved, was_generated) = map_output_offset(current, spans); + current = resolved; + generated = generated || was_generated; + } + (body.body_start + current, generated) +} + +/// The file span of compiled text written by the author, or `None` if generated. +pub(crate) fn authored_span(body: &ModelBody, start: usize, end: usize) -> Option<(usize, usize)> { + let (mapped_start, start_generated) = authored_offset(body, start); + if start_generated { + return None; + } + if end <= start { + return Some((mapped_start, mapped_start)); + } + let (mapped_last, last_generated) = authored_offset(body, end - 1); + if last_generated || mapped_last as i64 - mapped_start as i64 != (end - 1 - start) as i64 { + return None; + } + Some((mapped_start, mapped_last + 1)) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/constants.rs b/crates/sqlbuild-refactor/src/refactoring/constants.rs new file mode 100644 index 0000000000..e60ecdb0ad --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/constants.rs @@ -0,0 +1,83 @@ +//! Constants of project refactorings, mirroring `sqlbuild.compiler.refactoring.constants`. + +pub const MODEL_KIND_PREFIX: &str = "model:"; +pub const SQL_SUFFIX: &str = ".sql"; +pub const CURRENT_DIRECTORY: &str = "."; +pub const REF_FUNCTION: &str = "__ref"; +pub const REF_FIXTURE_PREFIX: &str = "__ref__"; +pub const EXPECTED_FIXTURE_PREFIX: &str = "__expected__"; +pub const PLACEHOLDER_PAD: char = '_'; +pub const GENERIC_PLACEHOLDER_KIND: char = 'x'; +pub const ROOT_SCOPE: &str = "root"; +pub const CTE_SCOPE_PREFIX: &str = "cte:"; +pub const MIGRATE_FROM_KEY: &str = "migrate_from"; +pub const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; +pub const COLUMNS_KEY: &str = "columns"; +pub const RELATIONSHIPS_AUDIT: &str = "relationships"; +pub const RELATIONSHIPS_TO_KEY: &str = "to"; +pub const RELATIONSHIPS_FIELD_KEY: &str = "field"; +pub const COLUMN_VALUED_CONFIG_KEYS: [&str; 9] = [ + "cursor", + "unique_key", + "updated_at", + "check_columns", + "merge_exclude_columns", + "partition_by", + "cluster_by", + "observed_at", + "row_diff_exclude_columns", +]; +pub const HISTORY_MATERIALIZATIONS: [&str; 2] = ["incremental", "snapshot"]; +pub const MIGRATABLE_MATERIALIZATIONS: [&str; 4] = ["table", "view", "incremental", "snapshot"]; +pub const COPIED_PROJECT_SUFFIXES: [&str; 7] = + [".sql", ".py", ".yml", ".yaml", ".toml", ".csv", ".json"]; +pub const IGNORED_PROJECT_DIRECTORIES: [&str; 5] = + ["target", "logs", "venv", "node_modules", "__pycache__"]; +pub const PYCACHE_DIRECTORY: &str = "__pycache__"; +pub const HIDDEN_PREFIX: &str = "."; +pub const HEADER_DESCRIPTION_KEY: &str = "description"; +pub const HEADER_INDENT: &str = " "; +pub const HEADER_SEPARATOR: &str = ","; +pub const HEADER_OPEN_PAREN: &str = "("; +pub const HEADER_OPENERS: [&str; 3] = ["(", "[", "{"]; +pub const HEADER_CLOSERS: [&str; 3] = [")", "]", "}"]; +pub const PARENTHESIZED_EMPTY_TOKENS: usize = 2; +pub const HEADER_KEY_AND_VALUE_TOKENS: usize = 2; +pub const REF_KIND: &str = "ref"; +pub const SOURCE_KIND: &str = "source"; +pub const SEED_KIND: &str = "seed"; +pub const MODEL_RESOURCE_TYPE: &str = "model"; +pub const UNKNOWN_COLUMN_TYPE: &str = "UNKNOWN"; +pub const GENERIC_DIALECT: &str = "generic"; +pub const SCHEMA_KEYWORD: &str = "SCHEMA"; +pub const MODEL_MANUAL_HELP: &str = "Pass the model into the macro as an argument, or edit the listed locations, then run the command again."; +pub const COLUMN_MANUAL_HELP: &str = "Edit the listed locations, then run the command again."; +/// `RefactorInputError`'s default code. +pub const INPUT_ERROR_CODE: &str = "C950"; +/// `RefactorEditError`'s code: two planned edits overlap. +pub const EDIT_ERROR_CODE: &str = "C957"; +/// `RefactorWriteError`'s code: project files changed between planning and writing. +pub const WRITE_ERROR_CODE: &str = "C958"; +/// Header whitespace that indents a MODEL `columns` entry. +pub const CONTRACT_HEADER_WHITESPACE: [char; 2] = [' ', '\t']; +pub const CONTRACT_DEFAULT_INDENT: &str = " "; +pub const CONTRACT_GENERATED_MARKER: &str = "generated"; +pub const CONTRACT_GENERATED_MARKER_LINES: usize = 5; +pub const CONTRACT_TYPE_KEY: &str = "type"; +pub const CONTRACT_YAML_COLUMNS_KEY: &str = "columns"; +pub const CONTRACT_YAML_LIST_NAME: &str = "- name:"; +pub const CONTRACT_YAML_COMMENT: &str = " #"; +pub const CONTRACT_MODEL_HEADER_ESCAPE: char = '\\'; +pub const CONTRACT_HEADER_WORD_EXTRA: [char; 2] = ['_', '$']; +pub const CONTRACT_QUOTES: [char; 2] = ['\'', '"']; +pub const CONTRACT_OPENERS: [char; 3] = ['(', '[', '{']; +pub const CONTRACT_CLOSERS: [char; 3] = [')', ']', '}']; +/// `ContractAdoptionError`'s message when two contract edits overlap. +pub const CONTRACT_OVERLAP_MESSAGE: &str = "overlapping contract source edits"; +pub const SHARED_SCHEMA_CONFLICT: &str = "model uses a shared SCHEMA declaration"; +pub const EXTERNAL_SCHEMA_CONFLICT: &str = "effective columns are owned by an external schema file"; +pub const MODEL_HEADER_CONFLICT: &str = "MODEL header could not be edited safely"; +pub const GENERATED_SOURCE_CONFLICT: &str = "source file is marked as generated"; +pub const SOURCE_BLOCK_CONFLICT: &str = "source declaration has no unambiguous YAML block"; +/// The `ValueError` prefix of a `name_pattern` the native matcher rejects. +pub const INVALID_NAME_PATTERN_PREFIX: &str = "invalid name_pattern: "; diff --git a/crates/sqlbuild-refactor/src/refactoring/errors.rs b/crates/sqlbuild-refactor/src/refactoring/errors.rs new file mode 100644 index 0000000000..2e4af91a60 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/errors.rs @@ -0,0 +1,62 @@ +//! Expected refactoring failures and the Python exception each becomes. + +use serde::Serialize; + +/// Which Python exception an expected refactoring failure becomes. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RefactorErrorKind { + /// `RefactorInputError`: the request cannot be planned. + Input, + /// `RefactorEditError`: two planned edits overlap. + Edit, + /// `RefactorWriteError`: project files changed while the refactoring ran. + Write, + /// An internal native failure, raised as `NativeCompilerError`. + Internal, + /// A `ValueError` the Python planner does not handle, such as a header the tokenizer rejects. + Value, + /// An `OSError` reading or writing project files, which Python does not handle either. + Io, +} + +/// An expected refactoring failure with Python's code, message and help. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RefactorError { + pub kind: RefactorErrorKind, + pub code: String, + pub message: String, + pub help: Option, +} + +impl RefactorError { + /// A `RefactorInputError` with an explicit code. + pub fn input(code: &str, message: impl Into, help: Option<&str>) -> Self { + Self { + kind: RefactorErrorKind::Input, + code: code.to_owned(), + message: message.into(), + help: help.map(str::to_owned), + } + } + + /// An unhandled `ValueError` with Python's message. + pub fn value(message: String) -> Self { + Self { + kind: RefactorErrorKind::Value, + code: String::new(), + message, + help: None, + } + } + + /// An internal native failure with the reason native code stopped. + pub fn internal(message: impl Into) -> Self { + Self { + kind: RefactorErrorKind::Internal, + code: String::new(), + message: message.into(), + help: None, + } + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/main/commit_refactor_plan.rs b/crates/sqlbuild-refactor/src/refactoring/main/commit_refactor_plan.rs new file mode 100644 index 0000000000..b6472727bf --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/main/commit_refactor_plan.rs @@ -0,0 +1,17 @@ +//! Write a verified refactoring plan into the project. + +use std::path::{Path, PathBuf}; + +use crate::refactoring::_helpers::files::workspace::commit_changes; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::RefactorPlan; +use crate::refactoring::types::Originals; + +/// Write every change, restoring all files if any write fails, and return the written paths. +pub fn commit_refactor_plan( + project_dir: &Path, + originals: &Originals, + plan: &RefactorPlan, +) -> Result, RefactorError> { + commit_changes(project_dir, originals, &plan.changes) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/main/edit_model_contract.rs b/crates/sqlbuild-refactor/src/refactoring/main/edit_model_contract.rs new file mode 100644 index 0000000000..0cfd19c490 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/main/edit_model_contract.rs @@ -0,0 +1,36 @@ +//! Contract adoption's metadata-preserving MODEL header edit. + +use sqlbuild_sqltext::compiler::main::model_header_columns::model_header_columns; + +use crate::refactoring::_helpers::contracts::model_edits::{ + columns_block, columns_edit, fixed_physical_columns, ownership_conflict, +}; +use crate::refactoring::_helpers::edits::header_edits::model_header_body_span; +use crate::refactoring::_helpers::scanning::chars::{chars, slice}; +use crate::refactoring::constants::MODEL_HEADER_CONFLICT; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{ContractEdit, ContractModelRequest}; + +/// Python's `edit_model`: add, retype or drop MODEL header columns to match the physical ones. +pub fn edit_model_contract(request: &ContractModelRequest) -> Result { + if let Some(conflict) = ownership_conflict(request) { + return Ok(conflict); + } + let physical = fixed_physical_columns(request)?; + let text = chars(&request.contents); + let body = model_header_body_span(&request.contents); + match model_header_columns(&request.contents).map_err(RefactorError::value)? { + Some(columns) => columns_edit(&text, request, &columns, &physical), + None => Ok(body.map_or_else( + || ContractEdit::Conflict(MODEL_HEADER_CONFLICT.to_owned()), + |(_, insertion)| { + ContractEdit::Edited(format!( + "{}{}{}", + slice(&text, 0, insertion), + columns_block(&physical), + slice(&text, insertion, text.len()) + )) + }, + )), + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/main/edit_source_contract.rs b/crates/sqlbuild-refactor/src/refactoring/main/edit_source_contract.rs new file mode 100644 index 0000000000..1dae822f6c --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/main/edit_source_contract.rs @@ -0,0 +1,32 @@ +//! Contract adoption's line-based YAML source declaration edit. + +use crate::refactoring::_helpers::contracts::source_edits::{ + column_edits, columns_insertion, columns_key, is_generated, +}; +use crate::refactoring::_helpers::contracts::yaml_lines::{lines_with_breaks, named_block}; +use crate::refactoring::constants::{GENERATED_SOURCE_CONFLICT, SOURCE_BLOCK_CONFLICT}; +use crate::refactoring::models::{ContractEdit, ContractSourceRequest}; + +/// Python's `edit_source`: add, retype or drop a source's YAML columns to match the physical ones. +pub fn edit_source_contract(request: &ContractSourceRequest) -> ContractEdit { + if is_generated(&request.discovered_contents) { + return ContractEdit::Conflict(GENERATED_SOURCE_CONFLICT.to_owned()); + } + let mut lines = lines_with_breaks(&request.contents); + let views: Vec<&str> = lines.iter().map(String::as_str).collect(); + let Some(block) = named_block(&views, &request.name) else { + return ContractEdit::Conflict(SOURCE_BLOCK_CONFLICT.to_owned()); + }; + let mut line_edits = match columns_key(&views, block) { + None => vec![(block.1, block.1, columns_insertion(request, block.2))], + Some(columns_index) => match column_edits(request, &views, (columns_index, block.1)) { + Ok(line_edits) => line_edits, + Err(conflict) => return ContractEdit::Conflict(conflict), + }, + }; + line_edits.sort_by(|left, right| right.cmp(left)); + for (start, end, replacement) in line_edits { + lines.splice(start..end, replacement); + } + ContractEdit::Edited(lines.concat()) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/main/mod.rs b/crates/sqlbuild-refactor/src/refactoring/main/mod.rs new file mode 100644 index 0000000000..7d817a2c92 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/main/mod.rs @@ -0,0 +1,6 @@ +pub mod commit_refactor_plan; +pub mod edit_model_contract; +pub mod edit_source_contract; +pub mod plan_refactor; +pub mod stage_refactor_plan; +pub mod with_model_migration; diff --git a/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs b/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs new file mode 100644 index 0000000000..4acace1b79 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/main/plan_refactor.rs @@ -0,0 +1,50 @@ +//! Plan a model rename, model move, or column rename from compiler facts. + +use crate::refactoring::_helpers::edits::text_edits::build_plan; +use crate::refactoring::_helpers::planning::column_planning::{ + column_rename_context, column_rename_parts, +}; +use crate::refactoring::_helpers::planning::model_planning::{model_parts, model_target}; +use crate::refactoring::_helpers::scanning::scan_context::ScanContext; +use crate::refactoring::constants::{COLUMN_MANUAL_HELP, MODEL_MANUAL_HELP}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::{RefactorFacts, RefactorOperation, RefactorPlan, RefactorRequest}; + +use crate::refactoring::types::DeclarationMoveHost; + +/// Plan one refactoring without touching files; `host` finds declaration-file moves. +pub fn plan_refactor( + facts: &RefactorFacts, + request: &RefactorRequest, + host: &DeclarationMoveHost<'_>, +) -> Result { + let context = ScanContext::for_facts(facts)?; + if request.operation == RefactorOperation::RenameColumn { + let column = column_rename_context(facts, request, &context)?; + let parts = column_rename_parts(&column)?; + let renamed: Vec<(String, String, String)> = parts + .cascaded + .iter() + .map(|model| (model.clone(), column.old.clone(), column.new.clone())) + .collect(); + return Ok(build_plan( + request.clone(), + parts, + &[], + (renamed, Some(COLUMN_MANUAL_HELP)), + )); + } + let target = model_target(facts, request)?; + let parts = model_parts(facts, &target, &context, host)?; + let mut moves = parts.moves.clone(); + let destination = target.destination(); + if destination != target.source_path { + moves.push((target.source_path.clone(), destination)); + } + Ok(build_plan( + target.request, + parts, + &moves, + (Vec::new(), Some(MODEL_MANUAL_HELP)), + )) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/main/stage_refactor_plan.rs b/crates/sqlbuild-refactor/src/refactoring/main/stage_refactor_plan.rs new file mode 100644 index 0000000000..88ff7ecdd0 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/main/stage_refactor_plan.rs @@ -0,0 +1,26 @@ +//! Stage a refactoring plan in a scratch copy of the project. + +use std::path::Path; + +use crate::refactoring::_helpers::files::workspace::{ + copy_project_inputs, read_originals, write_staged_changes, +}; +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::RefactorPlan; + +use crate::refactoring::types::Originals; + +/// Copy the project inputs once, apply the plan there, and return the original texts. +pub fn stage_refactor_plan( + project_dir: &Path, + staging_dir: &Path, + plan: &RefactorPlan, + copy_inputs: bool, +) -> Result { + if copy_inputs { + copy_project_inputs(project_dir, staging_dir)?; + } + let originals = read_originals(project_dir, &plan.changes)?; + write_staged_changes(staging_dir, &originals, &plan.changes)?; + Ok(originals) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/main/with_model_migration.rs b/crates/sqlbuild-refactor/src/refactoring/main/with_model_migration.rs new file mode 100644 index 0000000000..926fec7469 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/main/with_model_migration.rs @@ -0,0 +1,65 @@ +//! Add model `migrate_from` to a staged rename whenever its relation moves. + +use crate::refactoring::_helpers::edits::header_edits::insert_header_entry_edit; +use crate::refactoring::_helpers::planning::model_planning::decide_model_migration; +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::constants::MIGRATE_FROM_KEY; +use crate::refactoring::models::{ManualLocation, MigrationDeclaration, ModelFacts, RefactorPlan}; +use crate::refactoring::types::Originals; + +/// The plan with `migrate_from ` on the model whenever its relation moves. +pub fn with_model_migration( + plan: &RefactorPlan, + before: &[ModelFacts], + after: Option<&[ModelFacts]>, + originals: &Originals, +) -> RefactorPlan { + let old = plan.request.model_name.as_str(); + let new = plan.request.new_name.as_str(); + let decision = decide_model_migration(before, after, (old, new)); + let destination = plan.request.destination.clone(); + let mut migrated = plan.clone(); + if decision.blocked { + migrated.blocking.push(ManualLocation { + path: destination.unwrap_or_default(), + line: None, + column: None, + reason: decision.reason, + }); + return migrated; + } + let Some(index) = plan + .changes + .iter() + .position(|change| Some(&change.path) == destination.as_ref()) + else { + return migrated; + }; + if !decision.needed { + return migrated; + } + let declaration = format!("{MIGRATE_FROM_KEY} {old}"); + let change = &plan.changes[index]; + let contents = originals + .iter() + .find(|(path, _)| *path == change.original_path) + .map(|(_, text)| text.as_str()) + .unwrap_or_default(); + match insert_header_entry_edit(contents, &chars(contents), &declaration, &declaration) { + None => migrated.blocking.push(ManualLocation { + path: change.path.clone(), + line: Some(1), + column: Some(1), + reason: format!("add {declaration} to the MODEL header to keep history"), + }), + Some(edit) => { + migrated.changes[index].edits.push(edit); + migrated.migrations.push(MigrationDeclaration { + model_name: new.to_owned(), + declaration, + reason: decision.reason, + }); + } + } + migrated +} diff --git a/crates/sqlbuild-refactor/src/refactoring/mod.rs b/crates/sqlbuild-refactor/src/refactoring/mod.rs new file mode 100644 index 0000000000..65aaca2009 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-refactor/src/refactoring/models.rs b/crates/sqlbuild-refactor/src/refactoring/models.rs new file mode 100644 index 0000000000..e0336bfb20 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/models.rs @@ -0,0 +1,337 @@ +//! Plain-data refactoring facts and plans as Python's models hold them; offsets count code points. + +use serde::{Deserialize, Serialize}; + +use crate::refactoring::types::PhysicalColumn; + +/// One supported refactoring. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RefactorOperation { + RenameModel, + MoveModel, + RenameColumn, +} + +/// What one text edit changes, for output grouping. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EditKind { + Reference, + Fixture, + Header, + Column, + Migration, +} + +impl EditKind { + /// The kind's value in output, as Python's `EditKind` enum spells it. + pub fn value(self) -> &'static str { + match self { + Self::Reference => "reference", + Self::Fixture => "fixture", + Self::Header => "header", + Self::Column => "column", + Self::Migration => "migration", + } + } +} + +/// What an authored SQL file declares. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SqlFileRole { + Model, + Test, + Scenario, + Audit, + Hook, + Function, + Schema, + Yaml, +} + +/// One requested rename or move. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RefactorRequest { + pub operation: RefactorOperation, + pub model_name: String, + pub new_name: String, + #[serde(default)] + pub column_name: Option, + #[serde(default)] + pub destination: Option, + #[serde(default)] + pub cascade: bool, +} + +/// One replacement in an authored file, in offsets of the file before any edit. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TextEdit { + pub start: usize, + pub end: usize, + pub replacement: String, + pub kind: EditKind, + pub line: usize, + pub column: usize, + pub before: String, + pub after: String, +} + +impl TextEdit { + /// The edit with the display text Python's planner gives it instead of the defaults. + pub fn with_display(self, before: Option, after: Option) -> Self { + Self { + before: before.unwrap_or(self.before), + after: after.unwrap_or(self.after), + ..self + } + } +} + +/// Every edit to one file, and where the file lives afterwards. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FileChange { + pub path: String, + pub original_path: String, + pub edits: Vec, +} + +impl FileChange { + /// Whether the change moves the file. + pub fn moved(&self) -> bool { + self.path != self.original_path + } +} + +/// A location a refactoring cannot rewrite safely. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManualLocation { + pub path: String, + pub line: Option, + pub column: Option, + pub reason: String, +} + +/// A `migrate_from` declaration the refactoring adds to keep history. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct MigrationDeclaration { + pub model_name: String, + pub declaration: String, + pub reason: String, +} + +/// Every edit, manual location, and migration of one refactoring. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RefactorPlan { + pub request: RefactorRequest, + pub changes: Vec, + pub manual: Vec, + pub blocking: Vec, + pub migrations: Vec, + pub renamed_columns: Vec<(String, String, String)>, + pub help: Option, +} + +impl RefactorPlan { + /// The number of text edits the plan writes, over every changed file. + pub fn edit_count(&self) -> usize { + self.changes.iter().map(|change| change.edits.len()).sum() + } +} + +/// One rendered region of an expansion pass, pairing its source range with its output range. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExpansionSpan { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} + +/// Authored-to-expanded SQL evidence of one model file. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExpansionFacts { + pub expanded_sql: String, + pub passes: Vec>, +} + +/// The relation a model builds. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DestinationFacts { + pub database: Option, + pub schema: Option, + pub name: String, +} + +/// Where a column is declared. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LocationFacts { + pub path: String, + pub line: Option, + pub column: Option, +} + +/// One declared column of a model's schema entry. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SchemaColumnFacts { + pub name: String, + /// Whether the column declares a `migrate_from`. + pub migrate_from: bool, + pub location: Option, +} + +/// One dependency of a model. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DependencyFacts { + pub resource_type: String, + pub name: String, +} + +/// Every compiler fact of one model a refactoring reads. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ModelFacts { + pub name: String, + /// The project-relative POSIX path of the model file. + pub path: String, + pub deps: Vec, + pub query_sql: String, + pub authored_query_sql: String, + /// The authored `materialized` string config, as `get_config_str` returns it. + pub materialized: Option, + /// Whether `migrate_from` is a key of the model config. + pub declares_migrate_from: bool, + /// Whether the model config's `migrate_from` value is not `None`. + pub migrate_from_set: bool, + pub destination: DestinationFacts, + pub inferred_columns: Vec, + /// The columns of the model's schema entry, or `None` without a schema entry. + pub schema_columns: Option>, + /// The expansion of the model's file, if the compiler recorded one. + pub expansion: Option, +} + +/// The output columns of one source or seed. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResourceColumnsFacts { + pub name: String, + pub columns: Vec, +} + +/// One discovered file and its authored contents. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DiscoveredFile { + pub path: String, + pub contents: String, +} + +/// One discovered non-model SQL file with the SQL bodies discovery found in it, in file order. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AuthoredFile { + pub role: SqlFileRole, + pub path: String, + pub contents: String, + pub texts: Vec, +} + +/// Declaration files a model move takes along, and what blocks the move. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeclarationMoves { + pub moves: Vec<(String, String)>, + pub blocking: Vec, +} + +/// One scope-index declaration, keyed by its identity. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlacedDeclaration { + pub key: String, + /// `kind:name`. + pub label: String, + pub path: String, + pub line: Option, + pub column: Option, +} + +/// A declaration the scope placement moves, and the path it must move to. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RelocatedDeclaration { + pub key: String, + pub path: String, +} + +/// The scope placement answer for a model move across folders. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeclarationPlacement { + /// Relocated declarations in identity order, or `None` when placement is unsettled. + pub relocated: Option>, + /// Why the move preview failed, when it did. + pub diagnostics: Vec, + /// Every declaration of the scope index, in index order. + pub declarations: Vec, +} + +/// Every compiler fact of one project a refactoring plan reads. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RefactorFacts { + /// The resolved project directory. + pub project_dir: String, + /// The SQL analysis dialect, or `generic`. + pub dialect: String, + /// The `(major, minor)` Python version whose string semantics apply. + pub python_version: (u8, u8), + /// Python's `unicodedata.unidata_version`. + pub unicode_version: String, + pub models: Vec, + pub sources: Vec, + pub seeds: Vec, + /// Discovered model files, in discovery order. + pub model_files: Vec, + /// Discovered test, scenario, audit, hook, function and schema files, in Python's order. + pub authored_files: Vec, + /// Discovered source and seed YAML declaration files, in discovery order. + pub yaml_files: Vec, + /// Python strings naming the renamed model or column, found by the Python host. + pub python_locations: Vec, +} + +/// One authored model column: name, declared type and the path of its declaring file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ContractModelColumn { + pub name: String, + pub declared_type: Option, + pub location_path: Option, +} + +/// The model facts `edit_model` reads from a compiled model view. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ContractModelRequest { + pub contents: String, + pub relative_path: String, + pub shared_schema: bool, + pub columns: Vec, + /// Each dynamic family's optional `name_pattern`. + pub dynamic_patterns: Vec>, + pub physical_columns: Vec, + pub overwrite: bool, +} + +/// The source facts `edit_source` reads from a compiled source view. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ContractSourceRequest { + /// The text to edit: earlier edits of the same file, or the file as discovered. + pub contents: String, + /// The file as discovered, whose first lines may mark it as generated. + pub discovered_contents: String, + pub name: String, + /// Declared `(name, type)` columns. + pub columns: Vec<(String, Option)>, + pub physical_columns: Vec, + pub overwrite: bool, +} + +/// The edited text, or the ownership conflict that prevents a safe edit. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ContractEdit { + Edited(String), + Conflict(String), +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests.rs b/crates/sqlbuild-refactor/src/refactoring/tests.rs new file mode 100644 index 0000000000..b2ab52ea1a --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests.rs @@ -0,0 +1,20 @@ +#[path = "tests/test_column_references.rs"] +mod column_references; +#[path = "tests/test_contract_edits.rs"] +mod contract_edits; +#[path = "tests/test_declaration_edits.rs"] +mod declaration_edits; +#[path = "tests/test_header_edits.rs"] +mod header_edits; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_planning.rs"] +mod planning; +#[path = "tests/test_sql_sites.rs"] +mod sql_sites; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_text_edits.rs"] +mod text_edits; +#[path = "tests/test_yaml_edits.rs"] +mod yaml_edits; diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/helpers.rs b/crates/sqlbuild-refactor/src/refactoring/tests/helpers.rs new file mode 100644 index 0000000000..4e0755107d --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/helpers.rs @@ -0,0 +1,607 @@ +use std::collections::BTreeSet; +use std::fs; +use std::path::Path; + +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; +use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; + +use crate::refactoring::_helpers::edits::header_edits::{ + add_column_entry_edit, column_config_edits, column_entry_edits, consumer_column_header_edits, + header_tokens, insert_header_entry_edit, model_name_header_edits, schema_column_edits, + schema_model_name_edits, +}; +use crate::refactoring::_helpers::edits::text_edits::{apply_text_edits, file_changes, text_edit}; +use crate::refactoring::_helpers::edits::yaml_edits::{yaml_column_edits, yaml_model_edits}; +use crate::refactoring::_helpers::files::workspace::commit_changes; +use crate::refactoring::_helpers::planning::column_references::{ + BodyContext, BodyMapping, ColumnQuery, ResourceColumns, analyze_column, consumer_edits, +}; +use crate::refactoring::_helpers::planning::declaration_moves::declaration_moves; +use crate::refactoring::_helpers::planning::model_planning::{model_parts, model_target}; +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::_helpers::scanning::scan_context::ScanContext; +use crate::refactoring::_helpers::scanning::sql_sites::{ModelBody, analysis_sql, resource_sites}; +use crate::refactoring::errors::{RefactorError, RefactorErrorKind}; +use crate::refactoring::models::{ + DeclarationMoves, DeclarationPlacement, DestinationFacts, DiscoveredFile, EditKind, + ExpansionSpan, FileChange, ModelFacts, PlacedDeclaration, RefactorFacts, RefactorOperation, + RefactorRequest, RelocatedDeclaration, TextEdit, +}; +use crate::refactoring::tests::test_types::{ExpectedSpans, PlannedMoves, Spans}; + +pub(super) fn python() -> PythonText { + python_text((3, 12), "15.0.0").expect("Python 3.12 is supported") +} + +pub(super) fn context(dialect: &str) -> ScanContext { + ScanContext { + backtick_identifiers: quote_policy(dialect).backtick_identifiers, + python: python(), + } +} + +/// The text after applying edits. +pub(super) fn applied(text: &str, edits: &[TextEdit]) -> Result { + Ok(apply_text_edits(&chars(text), edits)?.into_iter().collect()) +} + +/// Each edit as `line:column kind before -> after`, the way the output tree shows it. +pub(super) fn described(edits: &[TextEdit]) -> Vec { + edits + .iter() + .map(|edit| { + format!( + "{}:{} {} {:?} -> {:?}", + edit.line, + edit.column, + edit.kind.value(), + edit.before, + edit.after + ) + }) + .collect() +} + +/// The texts of spans of `text`. +pub(super) fn span_texts(text: &str, spans: &[(usize, usize)]) -> Vec { + let characters: Vec = chars(text); + spans + .iter() + .map(|(start, end)| characters[*start..*end].iter().collect()) + .collect() +} + +/// Reference edits of `text` from `(start, end, replacement)` triples. +pub(super) fn reference_edits(text: &str, edits: &[(usize, usize, &'static str)]) -> Vec { + let characters: Vec = chars(text); + edits + .iter() + .map(|(start, end, replacement)| { + text_edit( + &characters, + (*start, *end), + (*replacement).to_owned(), + EditKind::Reference, + ) + }) + .collect() +} + +pub(super) fn owned_sites(sites: &[(usize, usize, &str)]) -> Vec<(usize, usize, String)> { + sites + .iter() + .map(|(start, end, name)| (*start, *end, (*name).to_owned())) + .collect() +} + +/// `(path, original path, edit count)` of each grouped change. +pub(super) fn grouped_changes( + edited_paths: &[&str], + moves: &[(&str, &str)], +) -> Vec<(String, String, usize)> { + let edit: TextEdit = reference_edits("abc", &[(0, 1, "x")]).remove(0); + let edits: Vec<(String, TextEdit)> = edited_paths + .iter() + .map(|path| ((*path).to_owned(), edit.clone())) + .collect(); + let moved: Vec<(String, String)> = moves + .iter() + .map(|(from, to)| ((*from).to_owned(), (*to).to_owned())) + .collect(); + file_changes(edits, &moved) + .into_iter() + .map(|change| (change.path, change.original_path, change.edits.len())) + .collect() +} + +pub(super) fn owned_changes(changes: &[(&str, &str, usize)]) -> Vec<(String, String, usize)> { + changes + .iter() + .map(|(path, original, count)| ((*path).to_owned(), (*original).to_owned(), *count)) + .collect() +} + +/// Each resource site as `kind name call-text name-text`. +pub(super) fn resource_site_lines(sql: &str) -> Vec { + resource_sites(&chars(sql), &context("duckdb")) + .into_iter() + .map(|site| { + let texts: Vec = span_texts( + sql, + &[(site.start, site.end), (site.name_start, site.name_end)], + ); + format!("{} {} {} {}", site.kind, site.name, texts[0], texts[1]) + }) + .collect() +} + +/// The analysis SQL and each table as `kind name placeholder,placeholder`. +pub(super) fn analysis_lines(sql: &str) -> (String, Vec) { + let analysis = analysis_sql(&chars(sql), &context("duckdb")); + let tables: Vec = analysis + .tables + .iter() + .map(|((kind, name), placeholders)| { + let joined: Vec = placeholders.iter().cloned().collect(); + format!("{kind} {name} {}", joined.join(",")) + }) + .collect(); + (analysis.sql, tables) +} + +/// A body at offset 10 whose one expansion turns `@m(x)` into a five-character call. +pub(super) fn expansion_body() -> ModelBody { + ModelBody { + body_start: 10, + compiled_sql: "SELECT a + b FROM t".to_owned(), + passes: vec![vec![ExpansionSpan { + source_start: 7, + source_end: 11, + output_start: 7, + output_end: 12, + }]], + } +} + +pub(super) fn consumer_model_edits(contents: &'static str) -> Result, RefactorError> { + model_name_header_edits(contents, &chars(contents), "stg_orders", "stg_order_lines") +} + +pub(super) fn owner_column_edits(contents: &'static str) -> Result, RefactorError> { + let text: Vec = chars(contents); + let tokens = header_tokens(contents, &text)?.unwrap_or_default(); + let mut edits: Vec = column_config_edits(&text, &tokens, "order_id", "order_key"); + edits.extend( + column_entry_edits(&text, &tokens, ("order_id", "order_key"), true).unwrap_or_default(), + ); + Ok(edits) +} + +pub(super) fn schema_model_edits(contents: &'static str) -> Result, RefactorError> { + schema_model_name_edits( + contents, + &chars(contents), + ("stg_orders", "stg_order_lines"), + python(), + ) +} + +pub(super) fn schema_field_edits(contents: &'static str) -> Result, RefactorError> { + schema_column_edits( + contents, + &chars(contents), + ("stg_orders", ("order_id", "order_key")), + python(), + ) +} + +/// The contents with a leading `migrate_from old_orders` entry, or `None` without a header. +pub(super) fn inserted_entry(contents: &str) -> Result, RefactorError> { + insert_header_entry_edit( + contents, + &chars(contents), + "migrate_from old_orders", + "migrate_from old_orders", + ) + .map(|edit| applied(contents, &[edit])) + .transpose() +} + +fn yaml_file(contents: &str) -> Vec { + vec![DiscoveredFile { + path: "sources/raw.yml".to_owned(), + contents: contents.to_owned(), + }] +} + +fn only_edits(edits: Vec<(String, TextEdit)>) -> Vec { + edits.into_iter().map(|(_, edit)| edit).collect() +} + +pub(super) fn yaml_reference_edits(contents: &'static str) -> Result, RefactorError> { + Ok(only_edits(yaml_model_edits( + &yaml_file(contents), + "stg_orders", + "stg_order_lines", + )?)) +} + +pub(super) fn yaml_field_edits(contents: &'static str) -> Result, RefactorError> { + Ok(only_edits(yaml_column_edits( + &yaml_file(contents), + "stg_orders", + "order_id", + "order_key", + )?)) +} + +/// `(start, end, before)` of renaming model `stg_orders`, then column `order_id` of it. +pub(super) fn yaml_spans(contents: &str) -> Result<(Spans, Spans), RefactorError> { + let models: Vec<(usize, usize, String)> = + spans(yaml_model_edits(&yaml_file(contents), "stg_orders", "x")?); + let columns: Vec<(usize, usize, String)> = spans(yaml_column_edits( + &yaml_file(contents), + "stg_orders", + "order_id", + "k", + )?); + Ok((models, columns)) +} + +fn spans(edits: Vec<(String, TextEdit)>) -> Vec<(usize, usize, String)> { + edits + .into_iter() + .map(|(_, edit)| (edit.start, edit.end, edit.before)) + .collect() +} + +pub(super) fn owned_span_pair(spans: (ExpectedSpans, ExpectedSpans)) -> (Spans, Spans) { + (owned_sites(spans.0), owned_sites(spans.1)) +} + +pub(super) fn model(name: &str, materialized: Option<&str>, schema: &str) -> ModelFacts { + ModelFacts { + name: name.to_owned(), + path: format!("models/{name}.sql"), + deps: Vec::new(), + query_sql: "SELECT 1".to_owned(), + authored_query_sql: "SELECT 1".to_owned(), + materialized: materialized.map(str::to_owned), + declares_migrate_from: false, + migrate_from_set: false, + destination: DestinationFacts { + database: Some("warehouse".to_owned()), + schema: Some(schema.to_owned()), + name: name.to_owned(), + }, + inferred_columns: Vec::new(), + schema_columns: None, + expansion: None, + } +} + +/// `model` placed in another database. +pub(super) fn model_in_database(name: &str, database: &str) -> ModelFacts { + let mut moved: ModelFacts = model(name, Some("table"), "analytics"); + moved.destination.database = Some(database.to_owned()); + moved +} + +pub(super) fn facts(project_dir: &str, models: Vec) -> RefactorFacts { + RefactorFacts { + project_dir: project_dir.to_owned(), + dialect: "duckdb".to_owned(), + python_version: (3, 12), + unicode_version: "15.0.0".to_owned(), + models, + sources: Vec::new(), + seeds: Vec::new(), + model_files: Vec::new(), + authored_files: Vec::new(), + yaml_files: Vec::new(), + python_locations: Vec::new(), + } +} + +/// A request on model `orders`. +pub(super) fn request( + operation: RefactorOperation, + new_name: &str, + destination: Option<&str>, +) -> RefactorRequest { + RefactorRequest { + operation, + model_name: "orders".to_owned(), + new_name: new_name.to_owned(), + column_name: None, + destination: destination.map(str::to_owned), + cascade: false, + } +} + +/// Each blocking item as `path: reason`. +pub(super) fn blocking_reasons( + facts: &RefactorFacts, + request: &RefactorRequest, +) -> Result, RefactorError> { + let target = model_target(facts, request)?; + let context = ScanContext { + backtick_identifiers: false, + python: python(), + }; + let host = |_: &str, _: &str| { + Ok(DeclarationPlacement { + relocated: Some(Vec::new()), + ..DeclarationPlacement::default() + }) + }; + let parts = model_parts(facts, &target, &context, &host)?; + Ok(parts + .blocking + .into_iter() + .map(|item| format!("{}: {}", item.path, item.reason)) + .collect()) +} + +/// The `(code, message)` a model request is refused with. +pub(super) fn refused_target( + facts: &RefactorFacts, + request: &RefactorRequest, +) -> Option<(String, String)> { + model_target(facts, request) + .err() + .map(|error| (error.code, error.message)) +} + +/// An edit replacing all of `text`. +pub(super) fn replace_all(text: &str, replacement: &str) -> TextEdit { + TextEdit { + start: 0, + end: text.chars().count(), + replacement: replacement.to_owned(), + kind: EditKind::Reference, + line: 1, + column: 1, + before: text.to_owned(), + after: replacement.to_owned(), + } +} + +pub(super) fn change(path: &str, original_path: &str, edits: Vec) -> FileChange { + FileChange { + path: path.to_owned(), + original_path: original_path.to_owned(), + edits, + } +} + +pub(super) fn owned_pairs(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(left, right)| ((*left).to_owned(), (*right).to_owned())) + .collect() +} + +/// Write `files` below `root`. +pub(super) fn write_files(root: &Path, files: &[(&str, &str)]) -> Result<(), RefactorError> { + files + .iter() + .try_for_each(|(path, contents)| { + let target = root.join(path); + fs::create_dir_all(target.parent().unwrap_or(root)) + .and_then(|()| fs::write(target, contents)) + }) + .map_err(|error| RefactorError::value(error.to_string())) +} + +/// Every file named in `files` as it is on disk now. +pub(super) fn files_on_disk(root: &Path, files: &[(&str, &str)]) -> Vec<(String, String)> { + files + .iter() + .map(|(path, _)| { + ( + (*path).to_owned(), + fs::read_to_string(root.join(path)).unwrap_or_default(), + ) + }) + .collect() +} + +/// `(kind, code, whether the message starts with prefix)` of a failed commit. +pub(super) fn commit_failure( + root: &Path, + originals: &[(&str, &str)], + changes: &[FileChange], + prefix: &str, +) -> Option<(RefactorErrorKind, String, bool)> { + commit_changes(root, &owned_pairs(originals), changes) + .err() + .map(|error| (error.kind, error.code, error.message.starts_with(prefix))) +} + +/// A scope-index declaration `kind:name` at `path`, line 1. +pub(super) fn placed(label: &str, path: &str) -> PlacedDeclaration { + PlacedDeclaration { + key: label.to_owned(), + label: label.to_owned(), + path: path.to_owned(), + line: Some(1), + column: Some(1), + } +} + +pub(super) fn relocated(label: &str, path: &str) -> RelocatedDeclaration { + RelocatedDeclaration { + key: label.to_owned(), + path: path.to_owned(), + } +} + +/// The moves and `path:line reason` blockers of moving model `orders` with a fixed placement. +pub(super) fn planned_declaration_moves( + root: &Path, + paths: (&str, &str), + placement: &DeclarationPlacement, +) -> Result { + let host = |_: &str, _: &str| Ok(placement.clone()); + let moves: DeclarationMoves = declaration_moves(root, "orders", paths, &host)?; + let blocking: Vec = moves + .blocking + .iter() + .map(|item| { + format!( + "{}:{} {}", + item.path, + item.line.unwrap_or_default(), + item.reason + ) + }) + .collect(); + Ok((moves.moves, blocking)) +} + +/// Rename `fact_orders.amount` in one consumer body: edited SQL, pass-through, manual reasons. +pub(super) fn plan_consumer( + sql: &str, + cascade: bool, +) -> Result<(String, bool, Vec), RefactorError> { + let text: Vec = chars(sql); + let analysis = analysis_sql(&text, &context("duckdb")); + let columns: ResourceColumns = vec![ + ( + ("ref".to_owned(), "fact_orders".to_owned()), + vec![ + "order_id".to_owned(), + "customer_id".to_owned(), + "amount".to_owned(), + ], + ), + ( + ("ref".to_owned(), "customers".to_owned()), + vec!["customer_id".to_owned(), "amount".to_owned()], + ), + ]; + let facts = analyze_column( + &analysis, + "duckdb", + &columns, + &ColumnQuery { + column: "amount", + target_tables: analysis.placeholders("ref", "fact_orders"), + target_ctes: BTreeSet::new(), + output_scopes: BTreeSet::new(), + }, + )?; + let body = BodyContext { + path: "models/consumer.sql", + contents: &text, + mapping: BodyMapping::Authored(0), + fallback_offset: 0, + }; + let result = consumer_edits(&facts, &body, ("amount", "revenue"), (cascade, cascade)); + let reasons: Vec = result.manual.into_iter().map(|item| item.reason).collect(); + Ok((applied(sql, &result.edits)?, result.passes_through, reasons)) +} + +/// Rename model `fact_orders` to `order_facts` in source and seed YAML. +pub(super) fn yaml_fact_model_edits( + contents: &'static str, +) -> Result, RefactorError> { + Ok(only_edits(yaml_model_edits( + &yaml_file(contents), + "fact_orders", + "order_facts", + )?)) +} + +/// Rename column `fact_orders.amount` to `revenue` in source and seed YAML. +pub(super) fn yaml_fact_column_edits( + contents: &'static str, +) -> Result, RefactorError> { + Ok(only_edits(yaml_column_edits( + &yaml_file(contents), + "fact_orders", + "amount", + "revenue", + )?)) +} + +pub(super) fn header_fact_model_edits( + contents: &'static str, +) -> Result, RefactorError> { + model_name_header_edits(contents, &chars(contents), "fact_orders", "order_facts") +} + +pub(super) fn header_fact_column_edits( + contents: &'static str, +) -> Result, RefactorError> { + consumer_column_header_edits( + contents, + &chars(contents), + "fact_orders", + ("amount", "revenue"), + ) +} + +pub(super) fn schema_fact_model_edits( + contents: &'static str, +) -> Result, RefactorError> { + schema_model_name_edits( + contents, + &chars(contents), + ("fact_orders", "order_facts"), + python(), + ) +} + +pub(super) fn schema_fact_column_edits( + contents: &'static str, +) -> Result, RefactorError> { + schema_column_edits( + contents, + &chars(contents), + ("fact_orders", ("amount", "revenue")), + python(), + ) +} + +/// The owner header edits renaming `amount` to `revenue` with a column migration. +pub(super) fn migrated_column_edits( + contents: &'static str, +) -> Result, RefactorError> { + let text: Vec = chars(contents); + let tokens = header_tokens(contents, &text)?.unwrap_or_default(); + let entries: Option> = + column_entry_edits(&text, &tokens, ("amount", "revenue"), true); + let added: Vec = entries.clone().unwrap_or_else(|| { + add_column_entry_edit(contents, &text, &tokens, ("amount", "revenue")) + .into_iter() + .collect() + }); + let mut edits: Vec = column_config_edits(&text, &tokens, "amount", "revenue"); + edits.extend(added); + Ok(edits) +} + +/// The header entry `migrate_from fact_orders` inserted at the top of the header. +pub(super) fn fact_orders_migration_entry( + contents: &'static str, +) -> Result, RefactorError> { + Ok(insert_header_entry_edit( + contents, + &chars(contents), + "migrate_from fact_orders", + "migrate_from fact_orders", + ) + .into_iter() + .collect()) +} + +/// Owned `(name, type)` physical columns. +pub(crate) fn physical_columns(columns: &[(&str, &str)]) -> Vec<(String, String)> { + columns + .iter() + .map(|(name, column_type)| ((*name).to_owned(), (*column_type).to_owned())) + .collect() +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_column_references.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_column_references.rs new file mode 100644 index 0000000000..2271d9a2c0 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_column_references.rs @@ -0,0 +1,102 @@ +use crate::refactoring::errors::RefactorError; +use crate::refactoring::tests::helpers::plan_consumer; +use crate::refactoring::tests::test_types::ColumnEditTestCase; + +#[test] +fn given_consumer_sql_when_planning_column_rename_then_edits_match() -> Result<(), RefactorError> { + let test_cases = [ + ColumnEditTestCase { + description: "bare projection keeps its output name", + sql: "SELECT o.amount FROM __ref(\"fact_orders\") AS o", + cascade: false, + expected_sql: "SELECT o.revenue AS amount FROM __ref(\"fact_orders\") AS o", + expected_passes_through: false, + expected_manual: None, + }, + ColumnEditTestCase { + description: "cascade renames a bare projection's output", + sql: "SELECT o.amount FROM __ref(\"fact_orders\") AS o", + cascade: true, + expected_sql: "SELECT o.revenue FROM __ref(\"fact_orders\") AS o", + expected_passes_through: true, + expected_manual: None, + }, + ColumnEditTestCase { + description: "aliased projection keeps its alias", + sql: "SELECT o.amount AS total FROM __ref(\"fact_orders\") AS o", + cascade: true, + expected_sql: "SELECT o.revenue AS total FROM __ref(\"fact_orders\") AS o", + expected_passes_through: false, + expected_manual: None, + }, + ColumnEditTestCase { + description: "filter reference is renamed", + sql: "SELECT order_id FROM __ref(\"fact_orders\") WHERE amount > 0", + cascade: false, + expected_sql: "SELECT order_id FROM __ref(\"fact_orders\") WHERE revenue > 0", + expected_passes_through: false, + expected_manual: None, + }, + ColumnEditTestCase { + description: "same-named column of another relation is untouched", + sql: "SELECT c.amount FROM __ref(\"fact_orders\") AS o JOIN __ref(\"customers\") AS c ON o.customer_id = c.customer_id", + cascade: false, + expected_sql: "SELECT c.amount FROM __ref(\"fact_orders\") AS o JOIN __ref(\"customers\") AS c ON o.customer_id = c.customer_id", + expected_passes_through: false, + expected_manual: None, + }, + ColumnEditTestCase { + description: "star through a CTE is followed", + sql: "WITH c AS (SELECT * FROM __ref(\"fact_orders\")) SELECT c.amount FROM c", + cascade: false, + expected_sql: "WITH c AS (SELECT * FROM __ref(\"fact_orders\")) SELECT c.revenue AS amount FROM c", + expected_passes_through: false, + expected_manual: None, + }, + ColumnEditTestCase { + description: "root star needs cascade", + sql: "SELECT * FROM __ref(\"fact_orders\")", + cascade: false, + expected_sql: "SELECT * FROM __ref(\"fact_orders\")", + expected_passes_through: false, + expected_manual: Some("rerun with --cascade"), + }, + ColumnEditTestCase { + description: "root star passes through under cascade", + sql: "SELECT * FROM __ref(\"fact_orders\")", + cascade: true, + expected_sql: "SELECT * FROM __ref(\"fact_orders\")", + expected_passes_through: true, + expected_manual: None, + }, + ColumnEditTestCase { + description: "USING join is left for the author", + sql: "SELECT o.order_id FROM __ref(\"fact_orders\") AS o JOIN __ref(\"customers\") AS c USING (amount)", + cascade: false, + expected_sql: "SELECT o.order_id FROM __ref(\"fact_orders\") AS o JOIN __ref(\"customers\") AS c USING (amount)", + expected_passes_through: false, + expected_manual: Some("USING or NATURAL join"), + }, + ]; + for test_case in test_cases { + let (sql, passes_through, reasons) = plan_consumer(test_case.sql, test_case.cascade)?; + let fragment: &str = test_case.expected_manual.unwrap_or_default(); + assert_eq!( + ( + sql.as_str(), + passes_through, + !reasons.is_empty(), + reasons.iter().all(|reason| reason.contains(fragment)), + ), + ( + test_case.expected_sql, + test_case.expected_passes_through, + test_case.expected_manual.is_some(), + true, + ), + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_contract_edits.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_contract_edits.rs new file mode 100644 index 0000000000..780f3acb04 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_contract_edits.rs @@ -0,0 +1,186 @@ +use crate::refactoring::_helpers::contracts::model_types::declared_type_span; +use crate::refactoring::_helpers::scanning::chars::{chars, find}; +use crate::refactoring::main::edit_model_contract::edit_model_contract; +use crate::refactoring::main::edit_source_contract::edit_source_contract; +use crate::refactoring::models::{ + ContractEdit, ContractModelColumn, ContractModelRequest, ContractSourceRequest, +}; +use crate::refactoring::tests::helpers::physical_columns; +use crate::refactoring::tests::test_types::{ + DeclaredTypeSpanTestCase, ModelContractTestCase, SourceContractTestCase, +}; + +const HEADER: &str = "MODEL (\n name m,\n columns (\n id (description 'key'),\n amount (audits [custom(type INT)], type DECIMAL(10, 2)),\n old (not_null true)\n )\n);\nSELECT 1"; +const SOURCES: &str = "sources:\n - name: orders\n columns:\n - name: id\n type: INT # key\n - name: gone\n type: TEXT\n - name: other\n"; + +#[test] +fn given_nested_type_key_when_locating_declared_type_then_returns_top_level_value() { + let test_cases = [ + DeclaredTypeSpanTestCase { + description: "a nested type argument precedes the declared column type", + contents: "amount (audits [custom(type INT)], type INT)", + declared_type: "INT", + expected_span: Some((40, 43)), + }, + DeclaredTypeSpanTestCase { + description: "a quoted type matches after unescaping", + contents: "amount (type 'a\\'b')", + declared_type: "a'b", + expected_span: Some((13, 19)), + }, + DeclaredTypeSpanTestCase { + description: "a type naming another value has no span", + contents: "amount (type TEXT)", + declared_type: "INT", + expected_span: None, + }, + ]; + + for test_case in test_cases { + let text = chars(test_case.contents); + let metadata = (find(&text, "(", 0).unwrap_or_default() + 1, text.len() - 1); + + let span = declared_type_span(&text, metadata, test_case.declared_type); + + assert_eq!(span, test_case.expected_span, "{}", test_case.description); + } +} + +#[test] +fn given_physical_columns_when_editing_model_contract_then_header_matches_python() { + let test_cases = [ + ModelContractTestCase { + description: "overwrite types, drops, renders and appends columns", + contents: HEADER, + columns: &[("id", None), ("amount", Some("DECIMAL(10,2)")), ("old", Some("INT"))], + dynamic_patterns: &[], + physical: &[ + ("ID", "BIGINT"), + ("amount", "DOUBLE"), + ("region_a", "TEXT"), + ("note", "STRUCT(a INT)"), + ], + overwrite: true, + expected_edit: ContractEdit::Edited( + "MODEL (\n name m,\n columns (\n id (type BIGINT, description 'key'),\n amount (audits [custom(type INT)], type DOUBLE),\n \n \n region_a (type TEXT)\n note (type \"STRUCT(a INT)\")\n )\n);\nSELECT 1".to_owned(), + ), + }, + ModelContractTestCase { + description: "dynamic family members are not frozen into the contract", + contents: HEADER, + columns: &[("id", None), ("amount", Some("DECIMAL(10,2)")), ("old", Some("INT"))], + dynamic_patterns: &[Some("region_.*")], + physical: &[("id", "INT"), ("region_a", "TEXT")], + overwrite: false, + expected_edit: ContractEdit::Edited(HEADER.replacen("id (description", "id (type INT, description", 1)), + }, + ModelContractTestCase { + description: "a header without columns gains a columns block", + contents: "MODEL (name m);\nSELECT 1", + columns: &[], + dynamic_patterns: &[], + physical: &[("id", "INT"), ("a b", "x\"y")], + overwrite: false, + expected_edit: ContractEdit::Edited( + "MODEL (name m\n columns (\n id (type INT)\n a b (type \"x\\\"y\")\n ));\nSELECT 1".to_owned(), + ), + }, + ModelContractTestCase { + description: "a declared column without an entry span is a conflict", + contents: HEADER, + columns: &[("missing", None)], + dynamic_patterns: &[], + physical: &[], + overwrite: false, + expected_edit: ContractEdit::Conflict( + "column 'missing' has no unambiguous authored span".to_owned(), + ), + }, + ]; + + for test_case in test_cases { + let request = ContractModelRequest { + contents: test_case.contents.to_owned(), + relative_path: "m.sql".to_owned(), + shared_schema: false, + columns: test_case + .columns + .iter() + .map(|(name, declared_type)| ContractModelColumn { + name: (*name).to_owned(), + declared_type: declared_type.map(str::to_owned), + location_path: None, + }) + .collect(), + dynamic_patterns: test_case + .dynamic_patterns + .iter() + .map(|pattern| pattern.map(str::to_owned)) + .collect(), + physical_columns: physical_columns(test_case.physical), + overwrite: test_case.overwrite, + }; + + let edit = edit_model_contract(&request); + + assert_eq!( + edit, + Ok(test_case.expected_edit), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_physical_columns_when_editing_source_contract_then_yaml_matches_python() { + let test_cases = [ + SourceContractTestCase { + description: "overwrite retypes with the comment kept, drops and appends", + contents: SOURCES, + columns: &[("id", Some("INT")), ("gone", Some("TEXT"))], + physical: &[("id", "BIGINT"), ("note", "it's")], + overwrite: true, + expected_edit: ContractEdit::Edited( + "sources:\n - name: orders\n columns:\n - name: id\n type: 'BIGINT' # key\n - name: note\n type: 'it''s'\n - name: other\n".to_owned(), + ), + }, + SourceContractTestCase { + description: "a source without columns gains a columns key", + contents: "sources:\n- name: orders\n schema: raw\n", + columns: &[], + physical: &[("id", "INT")], + overwrite: false, + expected_edit: ContractEdit::Edited( + "sources:\n- name: orders\n schema: raw\n columns:\n - name: id\n type: 'INT'\n".to_owned(), + ), + }, + SourceContractTestCase { + description: "a generated file is never edited", + contents: "# Generated file\nsources:\n- name: orders\n", + columns: &[], + physical: &[("id", "INT")], + overwrite: false, + expected_edit: ContractEdit::Conflict("source file is marked as generated".to_owned()), + }, + ]; + + for test_case in test_cases { + let request = ContractSourceRequest { + contents: test_case.contents.to_owned(), + discovered_contents: test_case.contents.to_owned(), + name: "orders".to_owned(), + columns: test_case + .columns + .iter() + .map(|(name, declared_type)| ((*name).to_owned(), declared_type.map(str::to_owned))) + .collect(), + physical_columns: physical_columns(test_case.physical), + overwrite: test_case.overwrite, + }; + + let edit = edit_source_contract(&request); + + assert_eq!(edit, test_case.expected_edit, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_declaration_edits.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_declaration_edits.rs new file mode 100644 index 0000000000..5d5877a592 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_declaration_edits.rs @@ -0,0 +1,132 @@ +use crate::refactoring::errors::RefactorError; +use crate::refactoring::tests::helpers::{ + applied, fact_orders_migration_entry, header_fact_column_edits, header_fact_model_edits, + migrated_column_edits, schema_fact_column_edits, schema_fact_model_edits, + yaml_fact_column_edits, yaml_fact_model_edits, +}; +use crate::refactoring::tests::test_types::ContentEditsTestCase; + +#[test] +fn given_declarations_when_renaming_then_python_planner_edits_are_kept() -> Result<(), RefactorError> +{ + let test_cases = [ + ContentEditsTestCase { + description: "relationships target written as a reference", + contents: "seeds:\n - name: customers\n columns:\n - name: customer_id\n audits:\n - relationships:\n to: __ref(\"fact_orders\")\n field: customer_id\n", + plan: yaml_fact_model_edits, + expected_contents: "seeds:\n - name: customers\n columns:\n - name: customer_id\n audits:\n - relationships:\n to: __ref(\"order_facts\")\n field: customer_id\n", + }, + ContentEditsTestCase { + description: "bare relationships target in a flow mapping", + contents: "audits:\n - relationships: {to: fact_orders, field: customer_id}\n", + plan: yaml_fact_model_edits, + expected_contents: "audits:\n - relationships: {to: order_facts, field: customer_id}\n", + }, + ContentEditsTestCase { + description: "reference inside an escaped double-quoted expression", + contents: "audits:\n - expression_is_true: {expression: \"id IN (SELECT id FROM __ref(\\\"fact_orders\\\"))\"}\n", + plan: yaml_fact_model_edits, + expected_contents: "audits:\n - expression_is_true: {expression: \"id IN (SELECT id FROM __ref(\\\"order_facts\\\"))\"}\n", + }, + ContentEditsTestCase { + description: "similar names and descriptions are left alone", + contents: "sources:\n - name: raw\n description: fact_orders feed\n audits:\n - relationships: {to: fact_orders_daily, field: id}\n", + plan: yaml_fact_model_edits, + expected_contents: "sources:\n - name: raw\n description: fact_orders feed\n audits:\n - relationships: {to: fact_orders_daily, field: id}\n", + }, + ContentEditsTestCase { + description: "YAML field of a relationship to the model", + contents: "audits:\n - relationships: {to: '__ref(\"fact_orders\")', field: amount}\n", + plan: yaml_fact_column_edits, + expected_contents: "audits:\n - relationships: {to: '__ref(\"fact_orders\")', field: revenue}\n", + }, + ContentEditsTestCase { + description: "YAML field of a relationship to another model", + contents: "audits:\n - relationships: {to: '__ref(\"customers\")', field: amount}\n", + plan: yaml_fact_column_edits, + expected_contents: "audits:\n - relationships: {to: '__ref(\"customers\")', field: amount}\n", + }, + ContentEditsTestCase { + description: "MODEL header quoted SQL and bare relationships target", + contents: "MODEL (\n audits [expression_is_true (expression \"id IN (SELECT id FROM __ref('fact_orders'))\")],\n columns (\n id (audits [relationships (to fact_orders, field id)]),\n ),\n);\nSELECT order_id FROM orders\n", + plan: header_fact_model_edits, + expected_contents: "MODEL (\n audits [expression_is_true (expression \"id IN (SELECT id FROM __ref('order_facts'))\")],\n columns (\n id (audits [relationships (to order_facts, field id)]),\n ),\n);\nSELECT order_id FROM orders\n", + }, + ContentEditsTestCase { + description: "MODEL header field of a referenced relationship target", + contents: "MODEL (\n columns (\n id (audits [relationships (to __ref(\"fact_orders\"), field amount)]),\n ),\n);\nSELECT order_id FROM orders\n", + plan: header_fact_column_edits, + expected_contents: "MODEL (\n columns (\n id (audits [relationships (to __ref(\"fact_orders\"), field revenue)]),\n ),\n);\nSELECT order_id FROM orders\n", + }, + ContentEditsTestCase { + description: "bare target and quoted SQL in every SCHEMA of a file", + contents: "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships (to fact_orders, field id)]),\n ),\n);\n\nSCHEMA (\n name order_totals,\n audits [expression_is_true (expression \"total > (SELECT 0 FROM __ref('fact_orders') LIMIT 1)\")],\n);\n", + plan: schema_fact_model_edits, + expected_contents: "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships (to order_facts, field id)]),\n ),\n);\n\nSCHEMA (\n name order_totals,\n audits [expression_is_true (expression \"total > (SELECT 0 FROM __ref('order_facts') LIMIT 1)\")],\n);\n", + }, + ContentEditsTestCase { + description: "SCHEMA field of a relationship to the model", + contents: "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships (to __ref(\"fact_orders\"), field amount)]),\n ),\n);\n", + plan: schema_fact_column_edits, + expected_contents: "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships (to __ref(\"fact_orders\"), field revenue)]),\n ),\n);\n", + }, + ContentEditsTestCase { + description: "declared column gains migrate_from before its metadata", + contents: "MODEL (\n columns (\n amount (nullable false),\n ),\n);\nSELECT order_id, amount FROM orders\n", + plan: migrated_column_edits, + expected_contents: "MODEL (\n columns (\n revenue (migrate_from amount, nullable false),\n ),\n);\nSELECT order_id, amount FROM orders\n", + }, + ContentEditsTestCase { + description: "bare declared column gains a migrate_from block", + contents: "MODEL (\n columns (amount),\n);\nSELECT order_id, amount FROM orders\n", + plan: migrated_column_edits, + expected_contents: "MODEL (\n columns (revenue (migrate_from amount)),\n);\nSELECT order_id, amount FROM orders\n", + }, + ContentEditsTestCase { + description: "undeclared column gets a new columns entry", + contents: "MODEL (\n materialized incremental,\n);\nSELECT order_id, amount FROM orders\n", + plan: migrated_column_edits, + expected_contents: "MODEL (\n columns (revenue (migrate_from amount)),\n materialized incremental,\n);\nSELECT order_id, amount FROM orders\n", + }, + ContentEditsTestCase { + description: "column-valued config follows the column", + contents: "MODEL (\n unique_key [order_id, amount],\n);\nSELECT order_id, amount FROM orders\n", + plan: migrated_column_edits, + expected_contents: "MODEL (\n columns (revenue (migrate_from amount)),\n unique_key [order_id, revenue],\n);\nSELECT order_id, amount FROM orders\n", + }, + ContentEditsTestCase { + description: "cursor_inputs keys and relationships targets follow the model", + contents: "MODEL (\n cursor_inputs (\n fact_orders ordered_at,\n ),\n columns (\n order_id (audits [relationships (to fact_orders, field order_id)]),\n ),\n);\nSELECT order_id, amount FROM orders\n", + plan: header_fact_model_edits, + expected_contents: "MODEL (\n cursor_inputs (\n order_facts ordered_at,\n ),\n columns (\n order_id (audits [relationships (to order_facts, field order_id)]),\n ),\n);\nSELECT order_id, amount FROM orders\n", + }, + ContentEditsTestCase { + description: "descriptions naming the model are left alone", + contents: "MODEL (\n description \"fact_orders summary\",\n);\nSELECT order_id, amount FROM orders\n", + plan: header_fact_model_edits, + expected_contents: "MODEL (\n description \"fact_orders summary\",\n);\nSELECT order_id, amount FROM orders\n", + }, + ContentEditsTestCase { + description: "multi-line header gets its own line", + contents: "MODEL (\n materialized table,\n);\nSELECT order_id, amount FROM orders\n", + plan: fact_orders_migration_entry, + expected_contents: "MODEL (\n migrate_from fact_orders,\n materialized table,\n);\nSELECT order_id, amount FROM orders\n", + }, + ContentEditsTestCase { + description: "empty header is expanded", + contents: "MODEL ();\nSELECT order_id, amount FROM orders\n", + plan: fact_orders_migration_entry, + expected_contents: "MODEL (\n migrate_from fact_orders,\n);\nSELECT order_id, amount FROM orders\n", + }, + ]; + for test_case in test_cases { + let edits = (test_case.plan)(test_case.contents)?; + assert_eq!( + applied(test_case.contents, &edits)?, + test_case.expected_contents, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_header_edits.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_header_edits.rs new file mode 100644 index 0000000000..d87bb0cb0d --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_header_edits.rs @@ -0,0 +1,108 @@ +use crate::refactoring::errors::RefactorError; +use crate::refactoring::tests::helpers::{ + applied, consumer_model_edits, described, inserted_entry, owner_column_edits, + schema_field_edits, schema_model_edits, +}; +use crate::refactoring::tests::test_types::{HeaderEditsTestCase, InsertEntryTestCase}; + +const CONSUMER: &str = "MODEL (\n description 'Reads __ref(\"stg_orders\") rows.',\n cursor_inputs (stg_orders (order_date)),\n relationships (to stg_orders, field order_id),\n materialized incremental,\n);\n\nSELECT 1\n"; +const OWNER: &str = "MODEL (\n materialized incremental,\n unique_key order_id,\n columns (\n order_id (description 'Order key.'),\n amount,\n ),\n);\n\nSELECT 1\n"; +const SCHEMA_FILE: &str = "SCHEMA (\n name orders_schema,\n relationships (to stg_orders, field order_id),\n);\nXSCHEMA (relationships (to stg_orders));\n"; + +#[test] +fn given_headers_when_renaming_then_token_exact_edits_are_planned() -> Result<(), RefactorError> { + let test_cases = [ + HeaderEditsTestCase { + description: "a renamed model in cursor_inputs, a bare relationships target and quoted SQL", + contents: CONSUMER, + plan: consumer_model_edits, + expected_edits: &[ + "3:18 header \"stg_orders\" -> \"stg_order_lines\"", + "4:21 header \"stg_orders\" -> \"stg_order_lines\"", + "2:29 reference \"stg_orders\" -> \"stg_order_lines\"", + ], + expected_contents: "MODEL (\n description 'Reads __ref(\"stg_order_lines\") rows.',\n cursor_inputs (stg_order_lines (order_date)),\n relationships (to stg_order_lines, field order_id),\n materialized incremental,\n);\n\nSELECT 1\n", + }, + HeaderEditsTestCase { + description: "a renamed key column with metadata gains migrate_from inside its parentheses", + contents: OWNER, + plan: owner_column_edits, + expected_edits: &[ + "3:14 header \"order_id\" -> \"order_key\"", + "5:5 header \"order_id\" -> \"order_key\"", + "5:15 migration \"\" -> \"migrate_from order_id\"", + ], + expected_contents: "MODEL (\n materialized incremental,\n unique_key order_key,\n columns (\n order_key (migrate_from order_id, description 'Order key.'),\n amount,\n ),\n);\n\nSELECT 1\n", + }, + HeaderEditsTestCase { + description: "a SCHEMA header target is renamed; a longer keyword is not a SCHEMA header", + contents: SCHEMA_FILE, + plan: schema_model_edits, + expected_edits: &["3:21 header \"stg_orders\" -> \"stg_order_lines\""], + expected_contents: "SCHEMA (\n name orders_schema,\n relationships (to stg_order_lines, field order_id),\n);\nXSCHEMA (relationships (to stg_orders));\n", + }, + HeaderEditsTestCase { + description: "a SCHEMA relationship field follows its upstream column", + contents: SCHEMA_FILE, + plan: schema_field_edits, + expected_edits: &["3:39 header \"order_id\" -> \"order_key\""], + expected_contents: "SCHEMA (\n name orders_schema,\n relationships (to stg_orders, field order_key),\n);\nXSCHEMA (relationships (to stg_orders));\n", + }, + ]; + for test_case in test_cases { + let edits = (test_case.plan)(test_case.contents)?; + assert_eq!( + described(&edits), + test_case.expected_edits, + "{}", + test_case.description + ); + assert_eq!( + applied(test_case.contents, &edits)?, + test_case.expected_contents, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_header_layouts_when_inserting_entry_then_python_spacing_is_kept() +-> Result<(), RefactorError> { + let test_cases = [ + InsertEntryTestCase { + description: "a multi-line header gains a first line with its indentation", + contents: "MODEL (\n materialized table,\n);\nSELECT 1", + expected_contents: Some( + "MODEL (\n migrate_from old_orders,\n materialized table,\n);\nSELECT 1", + ), + }, + InsertEntryTestCase { + description: "a one-line header gains a leading entry", + contents: "MODEL (materialized table);\nSELECT 1", + expected_contents: Some( + "MODEL (migrate_from old_orders, materialized table);\nSELECT 1", + ), + }, + InsertEntryTestCase { + description: "an empty header becomes a multi-line header", + contents: "MODEL ();\nSELECT 1", + expected_contents: Some("MODEL (\n migrate_from old_orders,\n);\nSELECT 1"), + }, + InsertEntryTestCase { + description: "a file without a header gets no edit", + contents: "SELECT 1", + expected_contents: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + inserted_entry(test_case.contents)?.as_deref(), + test_case.expected_contents, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_planning.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_planning.rs new file mode 100644 index 0000000000..c396b8eb57 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_planning.rs @@ -0,0 +1,326 @@ +use crate::refactoring::_helpers::planning::model_planning::decide_model_migration; +use crate::refactoring::errors::{RefactorError, RefactorErrorKind}; +use crate::refactoring::models::{DeclarationPlacement, RefactorOperation}; +use crate::refactoring::tests::helpers::{ + blocking_reasons, change, commit_failure, facts, files_on_disk, model, model_in_database, + owned_pairs, placed, planned_declaration_moves, refused_target, relocated, replace_all, + request, write_files, +}; +use crate::refactoring::tests::test_types::{ + CollisionTestCase, CommitTestCase, DeclarationMovesTestCase, MigrationDecisionTestCase, + RefusedTargetTestCase, +}; + +#[test] +fn given_taken_name_or_destination_when_planning_then_collisions_block() -> Result<(), RefactorError> +{ + let project = tempfile::tempdir().map_err(|error| RefactorError::value(error.to_string()))?; + write_files(project.path(), &[("models/finance/orders.sql", "SELECT 1")])?; + let project_facts = facts( + &project.path().to_string_lossy(), + vec![ + model("orders", Some("table"), "analytics"), + model("Order_Lines", Some("table"), "analytics"), + ], + ); + let test_cases = [ + CollisionTestCase { + description: "a rename onto another model's name, ignoring case", + request: request(RefactorOperation::RenameModel, "order_lines", None), + expected_reasons: &["models/Order_Lines.sql: model:order_lines already exists"], + }, + CollisionTestCase { + description: "a move onto an existing file", + request: request(RefactorOperation::MoveModel, "", Some("models/finance/")), + expected_reasons: &["models/finance/orders.sql: destination file already exists"], + }, + ]; + for test_case in test_cases { + assert_eq!( + blocking_reasons(&project_facts, &test_case.request)?, + test_case.expected_reasons, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_destination_outside_project_when_moving_then_request_is_refused() { + let project_facts = facts( + "/nonexistent/project", + vec![model("orders", Some("table"), "analytics")], + ); + let test_cases = [RefusedTargetTestCase { + description: "a destination above the project directory", + request: request( + RefactorOperation::MoveModel, + "", + Some("../elsewhere/orders.sql"), + ), + expected_error: ( + "C955", + "destination '../elsewhere/orders.sql' is outside the project", + ), + }]; + for test_case in test_cases { + assert_eq!( + refused_target(&project_facts, &test_case.request), + Some(( + test_case.expected_error.0.to_owned(), + test_case.expected_error.1.to_owned() + )), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relation_change_when_deciding_migration_then_python_decision_is_made() { + let before = vec![ + model("orders", Some("table"), "analytics"), + model("customers", Some("view"), "analytics"), + ]; + let test_cases = [ + MigrationDecisionTestCase { + description: "a renamed table keeps its history", + before: before.clone(), + after: Some(vec![ + model("order_lines", Some("table"), "analytics"), + model("customers", Some("view"), "analytics"), + ]), + expected_decision: ( + true, + false, + "keeps the relation's history and its old name working", + ), + }, + MigrationDecisionTestCase { + description: "an unknown materialization keeps no data", + before: vec![model("orders", None, "analytics")], + after: None, + expected_decision: (false, false, "'None' models keep no warehouse data"), + }, + MigrationDecisionTestCase { + description: "an edited project that does not compile still gets migrate_from", + before: before.clone(), + after: None, + expected_decision: ( + true, + false, + "keeps the relation's history; the destination could not be checked because the edited project does not compile", + ), + }, + MigrationDecisionTestCase { + description: "a move across databases is blocked", + before: before.clone(), + after: Some(vec![model_in_database("order_lines", "archive")]), + expected_decision: ( + false, + true, + "moves from database warehouse to archive; migrations cannot cross databases", + ), + }, + MigrationDecisionTestCase { + description: "an emptied schema is blocked", + before: before.clone(), + after: Some(vec![model("order_lines", Some("table"), "marts")]), + expected_decision: ( + false, + true, + "no model is left in schema analytics, so migrate_from orders cannot find the old relation; add a schema-qualified migrate_from for each target", + ), + }, + ]; + for test_case in test_cases { + let decision = decide_model_migration( + &test_case.before, + test_case.after.as_deref(), + ("orders", "order_lines"), + ); + assert_eq!( + (decision.needed, decision.blocked, decision.reason.as_str()), + test_case.expected_decision, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_failing_commit_when_committing_then_project_files_are_unchanged() +-> Result<(), RefactorError> { + let test_cases = [ + CommitTestCase { + description: "a write failing after an earlier write restores every file", + on_disk: &[ + ("models/orders.sql", "SELECT 1"), + ("models/customers.sql", "SELECT 2"), + ], + originals: &[ + ("models/orders.sql", "SELECT 1"), + ("models/customers.sql", "SELECT 2"), + ], + changes: vec![ + change( + "models/orders.sql", + "models/orders.sql", + vec![replace_all("SELECT 1", "SELECT 10")], + ), + change( + "models/orders.sql/customers.sql", + "models/customers.sql", + Vec::new(), + ), + ], + expected_error: (RefactorErrorKind::Io, "", ""), + expected_files: &[ + ("models/orders.sql", "SELECT 1"), + ("models/customers.sql", "SELECT 2"), + ], + }, + CommitTestCase { + description: "a file edited since planning writes nothing", + on_disk: &[("models/orders.sql", "SELECT 3")], + originals: &[("models/orders.sql", "SELECT 1")], + changes: vec![change( + "models/orders.sql", + "models/orders.sql", + vec![replace_all("SELECT 1", "SELECT 10")], + )], + expected_error: ( + RefactorErrorKind::Write, + "C958", + "files changed while the refactoring ran: models/orders.sql", + ), + expected_files: &[("models/orders.sql", "SELECT 3")], + }, + ]; + for test_case in test_cases { + let project = + tempfile::tempdir().map_err(|error| RefactorError::value(error.to_string()))?; + write_files(project.path(), test_case.on_disk)?; + let failure = commit_failure( + project.path(), + test_case.originals, + &test_case.changes, + test_case.expected_error.2, + ); + assert_eq!( + failure, + Some(( + test_case.expected_error.0, + test_case.expected_error.1.to_owned(), + true + )), + "{}", + test_case.description + ); + assert_eq!( + files_on_disk(project.path(), test_case.on_disk), + owned_pairs(test_case.expected_files), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_scope_placement_when_moving_across_folders_then_declaration_files_follow_or_block() +-> Result<(), RefactorError> { + let test_cases = [ + DeclarationMovesTestCase { + description: "a move within one folder asks no placement", + on_disk: &[], + paths: ("models/orders.sql", "models/orders_v2.sql"), + placement: DeclarationPlacement::default(), + expected_moves: &[], + expected_blocking: &[], + }, + DeclarationMovesTestCase { + description: "an unsettled placement blocks with the preview's reasons", + on_disk: &[], + paths: ("models/orders.sql", "models/marts/orders.sql"), + placement: DeclarationPlacement { + relocated: None, + diagnostics: vec!["Move source model:orders is unknown".to_owned()], + declarations: Vec::new(), + }, + expected_moves: &[], + expected_blocking: &["models/marts/orders.sql:0 Move source model:orders is unknown"], + }, + DeclarationMovesTestCase { + description: "an unsettled placement without reasons blocks with the default reason", + on_disk: &[], + paths: ("models/orders.sql", "models/marts/orders.sql"), + placement: DeclarationPlacement::default(), + expected_moves: &[], + expected_blocking: &[ + "models/marts/orders.sql:0 declaration placement at the destination could not be worked out", + ], + }, + DeclarationMovesTestCase { + description: "files move whole; split files, shared files, generated files and taken paths block", + on_disk: &[ + ("models/_sqlbuild/_macros/cents.py", "x"), + ("models/_sqlbuild/_enums/both.sql", "x"), + ("models/marts/_sqlbuild/_enums/both.sql", "x"), + ], + paths: ("models/orders.sql", "models/marts/orders.sql"), + placement: DeclarationPlacement { + relocated: Some(vec![ + relocated("enum:a", "models/marts/_sqlbuild/_enums/both.sql"), + relocated("enum:b", "models/other/_sqlbuild/_enums/both.sql"), + relocated("macro:cents", "models/marts/_sqlbuild/_macros/cents.py"), + relocated("macro:generated", "models/marts/_sqlbuild/_macros/gen.py"), + ]), + diagnostics: Vec::new(), + declarations: vec![ + placed("enum:a", "models/_sqlbuild/_enums/both.sql"), + placed("enum:b", "models/_sqlbuild/_enums/both.sql"), + placed("macro:cents", "models/_sqlbuild/_macros/cents.py"), + placed("macro:stays", "models/_sqlbuild/_macros/cents.py"), + placed("macro:generated", "models/_sqlbuild/_macros/gen.py"), + ], + }, + expected_moves: &[ + ( + "models/_sqlbuild/_enums/both.sql", + "models/marts/_sqlbuild/_enums/both.sql", + ), + ( + "models/_sqlbuild/_macros/cents.py", + "models/marts/_sqlbuild/_macros/cents.py", + ), + ], + expected_blocking: &[ + "models/_sqlbuild/_enums/both.sql:1 models/_sqlbuild/_enums/both.sql holds declarations that must move to different folders (models/marts/_sqlbuild/_enums/both.sql, models/other/_sqlbuild/_enums/both.sql); split the file first", + "models/_sqlbuild/_macros/gen.py:1 macro:generated must move to models/marts/_sqlbuild/_macros/gen.py, but it is not an authored project file", + "models/_sqlbuild/_macros/cents.py:1 models/marts/_sqlbuild/_macros/cents.py would take macro:stays along, which must stay in models/_sqlbuild/_macros/cents.py; split the file first", + "models/marts/_sqlbuild/_enums/both.sql:0 a declaration must move here, but the file already exists", + ], + }, + ]; + for test_case in test_cases { + let project = + tempfile::tempdir().map_err(|error| RefactorError::value(error.to_string()))?; + write_files(project.path(), test_case.on_disk)?; + let (moves, blocking) = + planned_declaration_moves(project.path(), test_case.paths, &test_case.placement)?; + assert_eq!( + moves, + owned_pairs(test_case.expected_moves), + "{}", + test_case.description + ); + assert_eq!( + blocking, test_case.expected_blocking, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_sql_sites.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_sql_sites.rs new file mode 100644 index 0000000000..4337764ecd --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_sql_sites.rs @@ -0,0 +1,203 @@ +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::_helpers::scanning::interpolation::interpolation_sites; +use crate::refactoring::_helpers::scanning::sql_sites::{ + authored_offset, authored_span, embedded_ref_spans, +}; +use crate::refactoring::tests::helpers::{ + analysis_lines, context, expansion_body, python, resource_site_lines, span_texts, +}; +use crate::refactoring::tests::test_types::{ + AnalysisSqlTestCase, AuthoredOffsetTestCase, AuthoredSpanTestCase, EmbeddedRefsTestCase, + InterpolationSitesTestCase, ResourceSitesTestCase, +}; + +#[test] +fn given_sql_when_scanning_interpolation_then_sites_outside_comments_and_strings_are_found() { + let test_cases = [ + InterpolationSitesTestCase { + description: "a reference call outside a comment", + dialect: "duckdb", + sql: "SELECT * FROM __ref('orders') -- __ref('ignored')\n", + expected_sites: &["__ref('orders')"], + }, + InterpolationSitesTestCase { + description: "a doubled quote stays inside its string", + dialect: "duckdb", + sql: "SELECT '__ref(''x'')' AS a, __source('raw', 'orders')", + expected_sites: &["__source('raw', 'orders')"], + }, + InterpolationSitesTestCase { + description: "macros, variables and templates", + dialect: "duckdb", + sql: "SELECT @cents(amount), @@limit_rows, ${var} FROM t", + expected_sites: &["@cents(amount)", "@@limit_rows", "${var}"], + }, + InterpolationSitesTestCase { + description: "a backslash before a line break backtracks to the last doubled quote", + dialect: "duckdb", + sql: "SELECT 'it''s \\\n' , __ref('a')", + expected_sites: &[], + }, + InterpolationSitesTestCase { + description: "dollar quotes hide calls, but not after an identifier character", + dialect: "duckdb", + sql: "SELECT $tag$__ref('x')$tag$, __ref('y'), x$y$__ref('z')", + expected_sites: &["__ref('y')", "__ref('z')"], + }, + InterpolationSitesTestCase { + description: "backticks quote where the dialect quotes identifiers with them", + dialect: "generic", + sql: "SELECT `__ref('x')`, __ref('y')", + expected_sites: &["__ref('y')"], + }, + InterpolationSitesTestCase { + description: "backticks are plain text in postgres", + dialect: "postgres", + sql: "SELECT `__ref('x')`, __ref('y')", + expected_sites: &["__ref('x')", "__ref('y')"], + }, + InterpolationSitesTestCase { + description: "a call left open is not a site", + dialect: "duckdb", + sql: "SELECT __ref('x' FROM t", + expected_sites: &[], + }, + ]; + for test_case in test_cases { + let scan = context(test_case.dialect); + let sites: Vec = + interpolation_sites(&chars(test_case.sql), scan.backtick_identifiers, python()) + .into_iter() + .map(|site| site.text) + .collect(); + assert_eq!(sites, test_case.expected_sites, "{}", test_case.description); + } +} + +#[test] +fn given_sql_when_finding_resource_sites_then_single_name_calls_are_returned() { + let test_cases = [ + ResourceSitesTestCase { + description: "spaced and double-quoted reference", + sql: "SELECT * FROM __ref( \"orders\" ) JOIN __seed('codes')", + expected_sites: &[ + "ref orders __ref( \"orders\" ) orders", + "seed codes __seed('codes') codes", + ], + }, + ResourceSitesTestCase { + description: "a two-argument source is not a resource site", + sql: "SELECT * FROM __source('raw', 'orders')", + expected_sites: &[], + }, + ]; + for test_case in test_cases { + assert_eq!( + resource_site_lines(test_case.sql), + test_case.expected_sites, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_when_building_analysis_sql_then_sites_become_same_length_placeholders() { + let test_cases = [ + AnalysisSqlTestCase { + description: "resource and macro sites", + sql: "SELECT a FROM __ref('orders') o JOIN @m(x) m", + expected_sql: "SELECT a FROM _qr0___________ o JOIN _qx1_ m", + expected_tables: &["ref orders _qr0___________"], + }, + AnalysisSqlTestCase { + description: "a site shorter than its placeholder base", + sql: "SELECT @m FROM t", + expected_sql: "SELECT _x FROM t", + expected_tables: &[], + }, + ]; + for test_case in test_cases { + let (sql, tables): (String, Vec) = analysis_lines(test_case.sql); + assert_eq!(sql, test_case.expected_sql, "{}", test_case.description); + assert_eq!( + tables, test_case.expected_tables, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_quoted_text_when_finding_embedded_refs_then_escaped_quotes_match() { + let test_cases = [EmbeddedRefsTestCase { + description: "an escaped double-quoted reference matches, a longer name does not", + text: "x __ref(\\\"orders\\\") __ref('orders_v2') y", + name: "orders", + expected_names: &["orders"], + }]; + for test_case in test_cases { + assert_eq!( + span_texts( + test_case.text, + &embedded_ref_spans(&chars(test_case.text), test_case.name) + ), + test_case.expected_names, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_expansion_passes_when_mapping_offsets_then_generated_text_is_flagged() { + let test_cases = [ + AuthoredOffsetTestCase { + description: "an offset before the expansion", + offset: 3, + expected_offset: (13, false), + }, + AuthoredOffsetTestCase { + description: "an offset inside generated text", + offset: 8, + expected_offset: (17, true), + }, + AuthoredOffsetTestCase { + description: "an offset after the expansion shifts back", + offset: 13, + expected_offset: (22, false), + }, + ]; + for test_case in test_cases { + assert_eq!( + authored_offset(&expansion_body(), test_case.offset), + test_case.expected_offset, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_expansion_passes_when_mapping_spans_then_generated_spans_have_no_source() { + let test_cases = [ + AuthoredSpanTestCase { + description: "a span after the expansion", + span: (13, 17), + expected_span: Some((22, 26)), + }, + AuthoredSpanTestCase { + description: "a span crossing generated text", + span: (6, 9), + expected_span: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + authored_span(&expansion_body(), test_case.span.0, test_case.span.1), + test_case.expected_span, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_text_edits.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_text_edits.rs new file mode 100644 index 0000000000..dd51eb9ad5 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_text_edits.rs @@ -0,0 +1,165 @@ +use crate::refactoring::_helpers::edits::text_edits::{ + identifier_sites, line_column, whole_word_offsets, +}; +use crate::refactoring::_helpers::scanning::chars::chars; +use crate::refactoring::tests::helpers::{ + applied, grouped_changes, owned_changes, owned_sites, reference_edits, +}; +use crate::refactoring::tests::test_types::{ + ApplyEditsTestCase, FileChangesTestCase, IdentifierSitesTestCase, LineColumnTestCase, + WholeWordTestCase, +}; + +#[test] +fn given_edits_when_applying_then_python_order_and_overlap_rules_hold() { + let test_cases = [ + ApplyEditsTestCase { + description: "disjoint edits apply back to front", + text: "SELECT a FROM t", + edits: &[(7, 8, "b"), (14, 15, "u")], + expected_text: Ok("SELECT b FROM u"), + }, + ApplyEditsTestCase { + description: "identical duplicates apply once", + text: "SELECT a FROM t", + edits: &[(7, 8, "b"), (7, 8, "b")], + expected_text: Ok("SELECT b FROM t"), + }, + ApplyEditsTestCase { + description: "overlapping edits are refused at the earlier edit", + text: "SELECT abc FROM t", + edits: &[(7, 10, "x"), (9, 12, "y")], + expected_text: Err("overlapping refactoring edits at offset 7"), + }, + ApplyEditsTestCase { + description: "two insertions at one offset keep Python's stable reverse order", + text: "x", + edits: &[(0, 0, "A"), (0, 0, "B")], + expected_text: Ok("BAx"), + }, + ApplyEditsTestCase { + description: "offsets count code points", + text: "é a", + edits: &[(2, 3, "b")], + expected_text: Ok("é b"), + }, + ]; + for test_case in test_cases { + let result: Result = applied( + test_case.text, + &reference_edits(test_case.text, test_case.edits), + ) + .map_err(|error| error.message); + assert_eq!( + result, + test_case + .expected_text + .map(str::to_owned) + .map_err(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_text_when_finding_identifier_sites_then_code_is_matched_ignoring_case() { + let test_cases = [ + IdentifierSitesTestCase { + description: "comments and strings are skipped, case is ignored", + text: "WITH __ref__orders AS (SELECT 1) SELECT * FROM __REF__ORDERS -- __ref__orders\n'__ref__orders'", + names: &["__ref__orders"], + expected_sites: &[(5, 18, "__ref__orders"), (47, 60, "__ref__orders")], + }, + IdentifierSitesTestCase { + description: "the longest name wins", + text: "x __ref__orders_v2 y", + names: &["__ref__orders", "__ref__orders_v2"], + expected_sites: &[(2, 18, "__ref__orders_v2")], + }, + IdentifierSitesTestCase { + description: "a name inside a longer identifier is not a site", + text: "a__ref__orders $__ref__orders __ref__orders$", + names: &["__ref__orders"], + expected_sites: &[], + }, + ]; + for test_case in test_cases { + let names: Vec = test_case + .names + .iter() + .map(|name| (*name).to_owned()) + .collect(); + assert_eq!( + identifier_sites(&chars(test_case.text), &names), + owned_sites(test_case.expected_sites), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_text_when_finding_whole_words_then_identifier_neighbours_are_excluded() { + let test_cases = [WholeWordTestCase { + description: "longer identifiers and case variants are not the word", + text: "amount, AMOUNT_total, x.amount", + word: "amount", + expected_offsets: &[0, 24], + }]; + for test_case in test_cases { + assert_eq!( + whole_word_offsets(&chars(test_case.text), test_case.word), + test_case.expected_offsets, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_offset_when_locating_then_line_and_column_are_one_based() { + let test_cases = [ + LineColumnTestCase { + description: "an offset after a line break", + text: "ab\ncd", + offset: 4, + expected_position: (2, 2), + }, + LineColumnTestCase { + description: "the first offset", + text: "ab\ncd", + offset: 0, + expected_position: (1, 1), + }, + ]; + for test_case in test_cases { + assert_eq!( + line_column(&chars(test_case.text), test_case.offset), + test_case.expected_position, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_edits_and_moves_when_grouping_then_changes_follow_final_paths() { + let test_cases = [FileChangesTestCase { + description: "a moved file's edits follow it and paths are sorted", + edited_paths: &["models/b.sql", "models/a.sql"], + moves: &[("models/b.sql", "models/z/b.sql")], + expected_changes: &[ + ("models/a.sql", "models/a.sql", 1), + ("models/z/b.sql", "models/b.sql", 1), + ], + }]; + for test_case in test_cases { + assert_eq!( + grouped_changes(test_case.edited_paths, test_case.moves), + owned_changes(test_case.expected_changes), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs new file mode 100644 index 0000000000..87f8028685 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_types.rs @@ -0,0 +1,212 @@ +use crate::refactoring::errors::{RefactorError, RefactorErrorKind}; +use crate::refactoring::models::{ContractEdit, DeclarationPlacement}; +use crate::refactoring::models::{FileChange, ModelFacts, RefactorRequest, TextEdit}; + +/// `(start, end, before)` of planned edits. +pub(super) type Spans = Vec<(usize, usize, String)>; + +/// Expected `(start, end, before)` of planned edits. +pub(super) type ExpectedSpans = &'static [(usize, usize, &'static str)]; + +/// Declaration file moves and `path:line reason` blockers. +pub(super) type PlannedMoves = (Vec<(String, String)>, Vec); + +/// Plans edits for one file's contents. +pub(super) type PlanEdits = fn(&'static str) -> Result, RefactorError>; + +pub(super) struct ApplyEditsTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + /// `(start, end, replacement)` in authoring order. + pub(super) edits: &'static [(usize, usize, &'static str)], + /// The edited text, or the overlap error message. + pub(super) expected_text: Result<&'static str, &'static str>, +} + +pub(super) struct IdentifierSitesTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) names: &'static [&'static str], + pub(super) expected_sites: &'static [(usize, usize, &'static str)], +} + +pub(super) struct WholeWordTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) word: &'static str, + pub(super) expected_offsets: &'static [usize], +} + +pub(super) struct LineColumnTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) offset: usize, + pub(super) expected_position: (usize, usize), +} + +pub(super) struct FileChangesTestCase { + pub(super) description: &'static str, + /// Each path gets one edit, in this order. + pub(super) edited_paths: &'static [&'static str], + pub(super) moves: &'static [(&'static str, &'static str)], + /// `(path, original path, edit count)`. + pub(super) expected_changes: &'static [(&'static str, &'static str, usize)], +} + +pub(super) struct InterpolationSitesTestCase { + pub(super) description: &'static str, + pub(super) dialect: &'static str, + pub(super) sql: &'static str, + pub(super) expected_sites: &'static [&'static str], +} + +pub(super) struct ResourceSitesTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// `kind name call-text name-text`. + pub(super) expected_sites: &'static [&'static str], +} + +pub(super) struct AnalysisSqlTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_sql: &'static str, + /// `kind name placeholder,placeholder`. + pub(super) expected_tables: &'static [&'static str], +} + +pub(super) struct EmbeddedRefsTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) name: &'static str, + pub(super) expected_names: &'static [&'static str], +} + +pub(super) struct AuthoredOffsetTestCase { + pub(super) description: &'static str, + pub(super) offset: usize, + pub(super) expected_offset: (usize, bool), +} + +pub(super) struct AuthoredSpanTestCase { + pub(super) description: &'static str, + pub(super) span: (usize, usize), + pub(super) expected_span: Option<(usize, usize)>, +} + +pub(super) struct HeaderEditsTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) plan: PlanEdits, + pub(super) expected_edits: &'static [&'static str], + pub(super) expected_contents: &'static str, +} + +pub(super) struct InsertEntryTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) expected_contents: Option<&'static str>, +} + +pub(super) struct YamlEditsTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) plan: PlanEdits, + pub(super) expected_contents: &'static str, +} + +pub(super) struct YamlSpansTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + /// `(start, end, before)` of each model rename edit, then of each column rename edit. + pub(super) expected_spans: (ExpectedSpans, ExpectedSpans), +} + +pub(super) struct CollisionTestCase { + pub(super) description: &'static str, + pub(super) request: RefactorRequest, + pub(super) expected_reasons: &'static [&'static str], +} + +pub(super) struct RefusedTargetTestCase { + pub(super) description: &'static str, + pub(super) request: RefactorRequest, + /// `(code, message)`. + pub(super) expected_error: (&'static str, &'static str), +} + +pub(super) struct MigrationDecisionTestCase { + pub(super) description: &'static str, + pub(super) before: Vec, + pub(super) after: Option>, + /// `(needed, blocked, reason)`. + pub(super) expected_decision: (bool, bool, &'static str), +} + +pub(super) struct CommitTestCase { + pub(super) description: &'static str, + /// Files on disk before the commit. + pub(super) on_disk: &'static [(&'static str, &'static str)], + /// The texts the plan was made from. + pub(super) originals: &'static [(&'static str, &'static str)], + pub(super) changes: Vec, + /// `(kind, code, message prefix)`. + pub(super) expected_error: (RefactorErrorKind, &'static str, &'static str), + /// Every on-disk file after the failed commit. + pub(super) expected_files: &'static [(&'static str, &'static str)], +} + +pub(super) struct DeclarationMovesTestCase { + pub(super) description: &'static str, + /// Files on disk below the project. + pub(super) on_disk: &'static [(&'static str, &'static str)], + /// `(source, destination)` of the moved model. + pub(super) paths: (&'static str, &'static str), + pub(super) placement: DeclarationPlacement, + pub(super) expected_moves: &'static [(&'static str, &'static str)], + /// `path:line reason` of each blocker. + pub(super) expected_blocking: &'static [&'static str], +} + +pub(super) struct ColumnEditTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) cascade: bool, + pub(super) expected_sql: &'static str, + pub(super) expected_passes_through: bool, + /// Text every manual reason contains, when the body has manual locations. + pub(super) expected_manual: Option<&'static str>, +} + +pub(super) struct ContentEditsTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) plan: PlanEdits, + pub(super) expected_contents: &'static str, +} + +pub(crate) struct DeclaredTypeSpanTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) declared_type: &'static str, + pub(crate) expected_span: Option<(usize, usize)>, +} + +pub(crate) struct ModelContractTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) columns: &'static [(&'static str, Option<&'static str>)], + pub(crate) dynamic_patterns: &'static [Option<&'static str>], + pub(crate) physical: &'static [(&'static str, &'static str)], + pub(crate) overwrite: bool, + pub(crate) expected_edit: ContractEdit, +} + +pub(crate) struct SourceContractTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) columns: &'static [(&'static str, Option<&'static str>)], + pub(crate) physical: &'static [(&'static str, &'static str)], + pub(crate) overwrite: bool, + pub(crate) expected_edit: ContractEdit, +} diff --git a/crates/sqlbuild-refactor/src/refactoring/tests/test_yaml_edits.rs b/crates/sqlbuild-refactor/src/refactoring/tests/test_yaml_edits.rs new file mode 100644 index 0000000000..05f83838e7 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/tests/test_yaml_edits.rs @@ -0,0 +1,73 @@ +use crate::refactoring::errors::RefactorError; +use crate::refactoring::tests::helpers::{ + applied, owned_span_pair, yaml_field_edits, yaml_reference_edits, yaml_spans, +}; +use crate::refactoring::tests::test_types::{YamlEditsTestCase, YamlSpansTestCase}; + +const SOURCES: &str = "sources:\n - name: raw_orders\n description: \"Feeds __ref('stg_orders').\"\n columns:\n - name: customer_id\n audits:\n relationships:\n to: stg_orders\n field: Order_ID\n"; + +#[test] +fn given_yaml_declarations_when_renaming_then_references_and_fields_are_edited() +-> Result<(), RefactorError> { + let test_cases = [ + YamlEditsTestCase { + description: "a quoted __ref and a bare relationships target are renamed", + contents: SOURCES, + plan: yaml_reference_edits, + expected_contents: "sources:\n - name: raw_orders\n description: \"Feeds __ref('stg_order_lines').\"\n columns:\n - name: customer_id\n audits:\n relationships:\n to: stg_order_lines\n field: Order_ID\n", + }, + YamlEditsTestCase { + description: "a relationships field of the renamed model's column matches ignoring case", + contents: SOURCES, + plan: yaml_field_edits, + expected_contents: "sources:\n - name: raw_orders\n description: \"Feeds __ref('stg_orders').\"\n columns:\n - name: customer_id\n audits:\n relationships:\n to: stg_orders\n field: order_key\n", + }, + ]; + for test_case in test_cases { + let edits = (test_case.plan)(test_case.contents)?; + assert_eq!( + applied(test_case.contents, &edits)?, + test_case.expected_contents, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_yaml_edge_cases_when_renaming_then_python_edits_are_kept() -> Result<(), RefactorError> { + let test_cases = [ + YamlSpansTestCase { + description: "a file PyYAML rejects is skipped", + contents: "sources: [unclosed\n", + expected_spans: (&[], &[]), + }, + YamlSpansTestCase { + description: "an anchored target is renamed through its alias", + contents: "a: &anchor stg_orders\nb: *anchor\nsources:\n - audits:\n relationships:\n to: *anchor\n", + expected_spans: (&[(11, 21, "stg_orders")], &[]), + }, + YamlSpansTestCase { + description: "an aliased string is edited once per visit and an aliased relationship is followed", + contents: "base: &rel\n to: stg_orders\n field: Order_ID\nsources:\n - d: &d \"x __ref('stg_orders')\"\n audits:\n relationships: *rel\n - e: *d\n", + expected_spans: ( + &[ + (75, 85, "stg_orders"), + (75, 85, "stg_orders"), + (17, 27, "stg_orders"), + ], + &[(37, 45, "Order_ID")], + ), + }, + ]; + for test_case in test_cases { + assert_eq!( + yaml_spans(test_case.contents)?, + owned_span_pair(test_case.expected_spans), + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-refactor/src/refactoring/types.rs b/crates/sqlbuild-refactor/src/refactoring/types.rs new file mode 100644 index 0000000000..fc910016d4 --- /dev/null +++ b/crates/sqlbuild-refactor/src/refactoring/types.rs @@ -0,0 +1,17 @@ +//! Callback and alias types shared by refactoring planning and its bindings. + +use crate::refactoring::errors::RefactorError; +use crate::refactoring::models::DeclarationPlacement; + +/// The shared scope placement for moving a model to a destination: `(model, destination)`. +pub type DeclarationMoveHost<'a> = + dyn Fn(&str, &str) -> Result + 'a; + +/// Original texts keyed by original path, in plan order. +pub type Originals = Vec<(String, String)>; + +/// One physical column: `(name, type)`. +pub type PhysicalColumn = (String, String); + +/// Replace lines `start..end` with new lines, as contract adoption's YAML source edit plans it. +pub type LineEdit = (usize, usize, Vec); diff --git a/crates/sqlbuild-render/Cargo.toml b/crates/sqlbuild-render/Cargo.toml index ace3d2d9c6..74d7b19ee8 100644 --- a/crates/sqlbuild-render/Cargo.toml +++ b/crates/sqlbuild-render/Cargo.toml @@ -10,6 +10,7 @@ rust-version.workspace = true [dependencies] sqlbuild-cache.workspace = true sqlbuild-core.workspace = true +serde_json.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-render/src/macro_calls/main/call_class_store_text.rs b/crates/sqlbuild-render/src/macro_calls/main/call_class_store_text.rs new file mode 100644 index 0000000000..6286af4934 --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/main/call_class_store_text.rs @@ -0,0 +1,21 @@ +/// `call_class_store_text`: the JSON naming a persistent call class by its tokens and relations. +pub fn call_class_store_text( + macro_tokens: &[String], + context_token: Option<&str>, + prior_relations: Option<&[(String, String)]>, +) -> String { + let relations: serde_json::Value = prior_relations.map_or(serde_json::Value::Null, |items| { + serde_json::Value::Array( + items + .iter() + .map(|(kind, name)| serde_json::json!([kind, name])) + .collect(), + ) + }); + serde_json::Value::Array(vec![ + serde_json::json!(macro_tokens), + serde_json::json!(context_token), + relations, + ]) + .to_string() +} diff --git a/crates/sqlbuild-render/src/macro_calls/main/mod.rs b/crates/sqlbuild-render/src/macro_calls/main/mod.rs index 46e047d83f..940094311c 100644 --- a/crates/sqlbuild-render/src/macro_calls/main/mod.rs +++ b/crates/sqlbuild-render/src/macro_calls/main/mod.rs @@ -1,3 +1,4 @@ +pub mod call_class_store_text; pub mod macro_call_names; pub mod scan_macro_call_sites; pub mod splice_macro_calls; diff --git a/crates/sqlbuild-render/src/macro_calls/tests.rs b/crates/sqlbuild-render/src/macro_calls/tests.rs index f84924210d..2a444d496e 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_call_class_store_text.rs"] +mod call_class_store_text; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_macro_call_memo.rs"] diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_call_class_store_text.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_call_class_store_text.rs new file mode 100644 index 0000000000..cdc5b99d9d --- /dev/null +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_call_class_store_text.rs @@ -0,0 +1,51 @@ +use crate::macro_calls::main::call_class_store_text::call_class_store_text; +use crate::macro_calls::tests::test_types::CallClassStoreTextTestCase; + +#[test] +fn given_call_class_inputs_when_naming_it_then_python_json_text_is_returned() { + let test_cases = [ + CallClassStoreTextTestCase { + description: "macros only", + context_token: None, + prior_relations: None, + expected_text: r#"[["cents"],null,null]"#, + }, + CallClassStoreTextTestCase { + description: "no relations passed yet differs from no typed reference text", + context_token: None, + prior_relations: Some(&[]), + expected_text: r#"[["cents"],null,[]]"#, + }, + CallClassStoreTextTestCase { + description: "relations and a context token", + context_token: Some("ctx"), + prior_relations: Some(&[("model", "orders")]), + expected_text: r#"[["cents"],"ctx",[["model","orders"]]]"#, + }, + CallClassStoreTextTestCase { + description: "non-ASCII text is kept and quotes and control characters are escaped", + context_token: Some("ü\"\n\u{1f}"), + prior_relations: Some(&[("source", "café")]), + expected_text: "[[\"cents\"],\"ü\\\"\\n\\u001f\",[[\"source\",\"café\"]]]", + }, + ]; + + for test_case in test_cases { + let relations: Option> = test_case.prior_relations.map(|items| { + items + .iter() + .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) + .collect() + }); + assert_eq!( + call_class_store_text( + &["cents".to_owned()], + test_case.context_token, + relations.as_deref() + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs index f6196dad72..9d4e68eeb4 100644 --- a/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs +++ b/crates/sqlbuild-render/src/macro_calls/tests/test_types.rs @@ -65,3 +65,10 @@ pub(crate) struct CallBoundsTestCase { pub(crate) expected_args_starts: &'static [usize], pub(crate) expected_failure: Option<(Option, Option<&'static str>, ScanError)>, } + +pub(crate) struct CallClassStoreTextTestCase { + pub(crate) description: &'static str, + pub(crate) context_token: Option<&'static str>, + pub(crate) prior_relations: Option<&'static [(&'static str, &'static str)]>, + pub(crate) expected_text: &'static str, +} diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index d9527daa60..bbb7d6c2f7 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -31,3 +31,4 @@ pub(crate) const EMPTY_FIXTURE_QUERY_TOKEN_COUNT: usize = 6; pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; pub(crate) const PROJECT_CONFIG_FILE: &str = "sqlbuild_project.toml"; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; +pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v1"; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index ad6787b3bd..86c9fa99f2 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -17,9 +17,9 @@ use fensu_policy::lifecycle::errors::LifecycleError; use fensu_policy::lifecycle::models::{ ApplySuppressionsRequest, ExactSuppression, Finding, FindingSeverity, ScopedIgnore, }; -use rayon::iter::{ - IndexedParallelIterator, IntoParallelIterator, IntoParallelRefIterator, ParallelIterator, -}; +#[cfg(test)] +use rayon::iter::IndexedParallelIterator; +use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; @@ -178,6 +178,7 @@ fn evaluate_pending_models( })) } +#[cfg(test)] pub(crate) fn evaluate_json(request_json: &str) -> Result { let request: EvaluateRequest = serde_json::from_str(request_json) .map_err(|error| format!("invalid rules request: {error}"))?; @@ -185,6 +186,7 @@ pub(crate) fn evaluate_json(request_json: &str) -> Result { } /// Decode a request whose models arrive as separate payloads, each with its exact digest. +#[cfg(test)] pub(crate) fn parse_parts( request_json: &[u8], model_jsons: &[&[u8]], @@ -202,6 +204,7 @@ pub(crate) fn parse_parts( Ok(request) } +#[cfg(test)] fn parse_model_payloads( model_jsons: &[&[u8]], model_digests: &[String], @@ -454,24 +457,21 @@ fn suppress_faults( apply_fault_policy(request, &evaluated_codes, faults) } -pub(crate) fn finalize_findings_json(request_json: &str) -> Result { - let input: crate::models::FinalizeFindingsRequest = serde_json::from_str(request_json) - .map_err(|error| format!("invalid findings request: {error}"))?; - if input.version != API_VERSION { - return Err(format!( - "unsupported rules native API version {}; expected {API_VERSION}", - input.version - )); - } - config::validate(&input.config)?; +/// Apply the exception policy once to completed built-in, SQL and custom findings. +pub(crate) fn finalize_findings( + project_dir: String, + config: crate::models::RulesConfig, + evaluated_codes: &[String], + findings: Vec, +) -> Result, String> { + config::validate(&config)?; let request = EvaluateRequest { - project_dir: input.project_dir, - config: input.config, + project_dir, + config, ..Default::default() }; validate_exception_paths(&request)?; - let faults = apply_fault_policy(&request, &input.evaluated_codes, input.findings)?; - serde_json::to_string(&faults).map_err(|error| error.to_string()) + apply_fault_policy(&request, evaluated_codes, findings) } fn apply_fault_policy( diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs new file mode 100644 index 0000000000..52219e9666 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -0,0 +1,121 @@ +//! Evaluate a natively built rules request, reusing the last identical request's response. + +use std::path::{Path, PathBuf}; + +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::constants::{API_VERSION, NATIVE_ROWS_MEMO_VERSION}; +use crate::engine::_helpers::evaluation::evaluate_request; +use crate::errors::RowsError; +use crate::models::{EvaluateRequest, Model, RowsEvaluation, RowsRequest}; + +pub(crate) fn evaluate_rows(rows: RowsRequest) -> Result { + let RowsRequest { mut request, memo } = rows; + let models: Vec = std::mem::take(&mut request.models) + .into_iter() + .map(digested) + .collect::>()?; + let identity: Option<(String, PathBuf)> = match memo { + Some((code_identity, path)) => Some((identity(&request, &models, &code_identity)?, path)), + None => None, + }; + let reused: Option = identity + .as_ref() + .and_then(|(identity, path)| read_response(path, identity)); + let (response, was_reused): (String, bool) = match reused { + Some(response) => (response, true), + None => { + request.models = models; + let response: String = evaluate_request(request).map_err(RowsError::Rules)?; + if let Some((identity, path)) = &identity { + write_response(path, identity, &response).map_err(RowsError::Io)?; + } + (response, false) + } + }; + decode(&response, was_reused) +} + +/// `model` with the digest of its exact facts, which keys its cached findings. +fn digested(mut model: Model) -> Result { + let bytes: Vec = serde_json::to_vec(&model).map_err(rules_error)?; + model.payload_digest = Some(format!("{:x}", Sha256::digest(&bytes))); + Ok(model) +} + +/// The memo version, build, model-free request and every model's digest, in order. +fn identity( + request: &EvaluateRequest, + models: &[Model], + code_identity: &str, +) -> Result { + let mut hasher: Sha256 = Sha256::new(); + for part in [NATIVE_ROWS_MEMO_VERSION, code_identity] { + hasher.update(part.as_bytes()); + hasher.update(b"\0"); + } + hasher.update(serde_json::to_vec(request).map_err(rules_error)?); + hasher.update(b"\0"); + hasher.update((models.len() as u64).to_le_bytes()); + for model in models { + hasher.update( + model + .payload_digest + .as_deref() + .unwrap_or_default() + .as_bytes(), + ); + hasher.update(b"\0"); + } + Ok(format!("{:x}", hasher.finalize())) +} + +/// The stored response for exactly `identity`; an unreadable or foreign memo answers nothing. +fn read_response(path: &Path, identity: &str) -> Option { + let Ok(bytes) = std::fs::read(path) else { + return None; + }; + let Ok(stored) = serde_json::from_slice::(&bytes) else { + return None; + }; + if stored.get("identity")?.as_str()? != identity { + return None; + } + stored.get("response")?.as_str().map(str::to_owned) +} + +/// Atomically replace the memo with the latest evaluated request's response. +fn write_response(path: &Path, identity: &str, response: &str) -> Result<(), std::io::Error> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let temporary: PathBuf = path.with_extension(format!( + "tmp-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + std::fs::write( + &temporary, + serde_json::to_vec(&serde_json::json!({"identity": identity, "response": response}))?, + )?; + std::fs::rename(&temporary, path) +} + +fn decode(response: &str, reused: bool) -> Result { + let value: Value = serde_json::from_str(response).map_err(rules_error)?; + if value.get("version").and_then(Value::as_u64) != Some(u64::from(API_VERSION)) { + return Err(RowsError::Rules( + "native rules engine returned an unsupported response version".to_owned(), + )); + } + let mut evaluation: RowsEvaluation = serde_json::from_value(value).map_err(|_| { + RowsError::Rules("native rules engine returned invalid findings".to_owned()) + })?; + evaluation.reused = reused; + Ok(evaluation) +} + +fn rules_error(error: serde_json::Error) -> RowsError { + RowsError::Rules(error.to_string()) +} diff --git a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs index ed7464e6d1..b82023ac5c 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs @@ -1,3 +1,6 @@ pub mod cache; pub mod custom_host; pub mod evaluation; +pub(crate) mod memo; +pub(crate) mod project_rows; +pub(crate) mod rows; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs new file mode 100644 index 0000000000..0ed0d546a9 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -0,0 +1,102 @@ +//! Rules model rows read from the compile's natively owned project facts. + +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, DeclarationFact, ModelSchemaFact, +}; + +use crate::models::{ + Declaration, DeclarationMember, DynamicFamilyRow, DynamicProofRow, ModelRow, Reference, + SchemaColumnRow, SchemaRow, +}; + +/// The row the rules builder reads for one model; the authored query wins over the whole file. +pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result { + if let Some(error) = &facts.config.encoding_error { + return Err(error.clone()); + } + let analysis = facts.analysis.clone().unwrap_or_default(); + Ok(ModelRow { + name: facts.name.clone(), + relative_path: facts.relative_path.clone(), + query_sql: facts.query_sql.clone(), + authored_sql: if facts.authored_query_sql.is_empty() { + facts.authored_sql.clone() + } else { + facts.authored_query_sql.clone() + }, + config: facts.config.values.clone(), + authored_config_keys: facts.config.header_keys.clone(), + logical_schema: facts.config.layer_schema.clone(), + references: facts + .references + .iter() + .map(|reference| Reference { + ref_kind: reference.kind.clone(), + ref_name: reference.name.clone(), + ref_package: reference.package.clone(), + }) + .collect(), + schema: facts.schema.as_ref().map(schema_row), + inferred_columns: analysis.inferred_columns, + dynamic_proof: analysis.dynamic_proof.map(|proof| DynamicProofRow { + output_proven: proof.output_proven, + bare_dynamic_pivot: proof.bare_dynamic_pivot, + families: proof.families, + }), + enum_columns: facts.enum_columns.clone(), + enum_declarations: facts.enum_declarations.iter().map(declaration).collect(), + constant_declarations: facts + .constant_declarations + .iter() + .map(declaration) + .collect(), + }) +} + +fn schema_row(schema: &ModelSchemaFact) -> SchemaRow { + SchemaRow { + audit_count: schema.audit_count, + columns: schema + .columns + .iter() + .map(|column| SchemaColumnRow { + name: column.name.clone(), + data_type: column.data_type.clone(), + nullable: column.nullable, + audit_count: column.audit_count, + }) + .collect(), + dynamic_columns: schema + .dynamic_columns + .iter() + .map(|family| DynamicFamilyRow { + key: family.key.clone(), + name: family.name.clone(), + pivot_column: family.pivot_column.clone(), + value_column: family.value_column.clone(), + aggregate: family.aggregate.clone(), + data_type: family.data_type.clone(), + name_pattern: family.name_pattern.clone(), + }) + .collect(), + } +} + +/// A declaration as the rules request carries it. +pub(crate) fn declaration(fact: &DeclarationFact) -> Declaration { + Declaration { + name: fact.name.clone(), + relative_path: fact.relative_path.clone(), + members: fact + .members + .iter() + .map(|(name, value)| DeclarationMember { + name: name.clone(), + value: value.clone(), + }) + .collect(), + value: fact.value.clone(), + value_type: fact.value_type.clone(), + render_as: fact.render_as.clone(), + } +} diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs new file mode 100644 index 0000000000..4058e6b6f2 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -0,0 +1,200 @@ +//! Build rules request models from compiled-model rows, with their type proofs. + +use std::cell::RefCell; +use std::collections::HashMap; + +use sqlbuild_analysis::type_system::errors::TypeNormalizationError; +use sqlbuild_analysis::type_system::main::normalize_type::normalize_type; +use sqlbuild_analysis::type_system::models::NormalizedType; + +use crate::models::{ + Column, DynamicColumnFamily, Model, ModelRow, ModelRowsContext, SchemaRow, SqlTestMode, +}; +use crate::types::TypeEquality; + +/// The project counts and type comparer one model is built with. +struct ModelInputs<'a, 'c> { + context: ModelRowsContext<'a>, + comparer: &'c Comparer<'a>, + compiled_audit_count: u32, + targeting_test_count: u32, +} + +pub(crate) fn build_models( + rows: Vec, + context: ModelRowsContext<'_>, + mut fallback: impl FnMut(&str, &str) -> Result, +) -> Result, String> { + let mut audit_counts: HashMap<&str, u32> = HashMap::new(); + for target in context.attached_audit_targets { + *audit_counts.entry(target.as_str()).or_default() += 1; + } + let mut test_counts: HashMap<&str, u32> = HashMap::new(); + for test in context.sql_tests { + if matches!(test.mode, SqlTestMode::Model) { + for name in &test.target_model_names { + *test_counts.entry(name.as_str()).or_default() += 1; + } + } + } + let comparer: Comparer<'_> = Comparer { + dialect: context.dialect, + normalized: RefCell::new(HashMap::new()), + fallback: RefCell::new(&mut fallback), + }; + rows.into_iter() + .map(|row| { + let inputs: ModelInputs<'_, '_> = ModelInputs { + context, + comparer: &comparer, + compiled_audit_count: audit_counts.get(row.name.as_str()).copied().unwrap_or(0), + targeting_test_count: test_counts.get(row.name.as_str()).copied().unwrap_or(0), + }; + build_model(row, &inputs) + }) + .collect() +} + +fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result { + let ModelInputs { + context, + compiled_audit_count, + targeting_test_count, + .. + } = *inputs; + let sql_analysis_disabled: bool = !context.sql_analysis_enabled + || row.config.get("sql_analysis") == Some(&serde_json::Value::Bool(false)); + let bare_dynamic_pivot: bool = row + .dynamic_proof + .as_ref() + .is_some_and(|proof| proof.bare_dynamic_pivot); + let (columns, dynamic_columns, dynamic_columns_proven, schema_audit_count) = match &row.schema { + Some(schema) => schema_facts(&row, schema, inputs)?, + None => (Vec::new(), Vec::new(), false, 0), + }; + Ok(Model { + sql_analysis_disabled, + name: row.name, + relative_path: row.relative_path, + query_sql: row.query_sql, + authored_sql: row.authored_sql, + config: row.config, + authored_config_keys: row.authored_config_keys, + logical_schema: row.logical_schema, + references: row.references, + columns, + dynamic_columns, + dynamic_columns_proven, + bare_dynamic_pivot, + enum_columns: row.enum_columns, + enum_declarations: row.enum_declarations, + constant_declarations: row.constant_declarations, + declared_audit_count: schema_audit_count.max(compiled_audit_count), + targeting_test_count, + empty_input_only_test_count: 0, + payload_digest: None, + }) +} + +type SchemaFacts = (Vec, Vec, bool, u32); + +fn schema_facts( + row: &ModelRow, + schema: &SchemaRow, + inputs: &ModelInputs<'_, '_>, +) -> Result { + let ModelInputs { + context, comparer, .. + } = *inputs; + let inferred: HashMap<&str, Option<&str>> = row + .inferred_columns + .iter() + .map(|(name, data_type)| (name.as_str(), data_type.as_deref())) + .collect(); + let mut columns: Vec = Vec::with_capacity(schema.columns.len()); + for column in &schema.columns { + let declared: &str = column.data_type.as_deref().unwrap_or_default(); + let inferred_type: Option<&str> = inferred.get(column.name.as_str()).copied().flatten(); + let type_proven: bool = match inferred_type { + Some(inferred_type) + if context.include_type_proof + && !declared.is_empty() + && !inferred_type.is_empty() => + { + comparer.equal(declared, inferred_type)? + } + _ => false, + }; + columns.push(Column { + name: column.name.clone(), + data_type: declared.to_owned(), + nullable: column.nullable, + audit_count: column.audit_count, + type_proven, + }); + } + let audit_count: u32 = schema.audit_count + + schema + .columns + .iter() + .map(|column| column.audit_count) + .sum::(); + if schema.dynamic_columns.is_empty() { + return Ok((columns, Vec::new(), false, audit_count)); + } + let proof_types: HashMap<&str, Option<&str>> = row + .dynamic_proof + .iter() + .flat_map(|proof| &proof.families) + .map(|(key, inferred_type)| (key.as_str(), inferred_type.as_deref())) + .collect(); + let mut dynamic_columns: Vec = Vec::new(); + for family in &schema.dynamic_columns { + let inferred_type: Option<&str> = proof_types.get(family.key.as_str()).copied().flatten(); + let type_proven: bool = match inferred_type { + Some(inferred_type) if context.include_type_proof && !inferred_type.is_empty() => { + comparer.equal(&family.data_type, inferred_type)? + } + _ => false, + }; + dynamic_columns.push(DynamicColumnFamily { + name: family.name.clone(), + pivot_column: family.pivot_column.clone(), + value_column: family.value_column.clone(), + aggregate: family.aggregate.clone(), + data_type: family.data_type.clone(), + name_pattern: family.name_pattern.clone(), + type_proven, + }); + } + let proven: bool = row + .dynamic_proof + .as_ref() + .is_some_and(|proof| proof.output_proven); + Ok((columns, dynamic_columns, proven, audit_count)) +} + +/// Python's `types_equal`, normalizing each type once; Python raises what normalization rejects. +struct Comparer<'a> { + dialect: &'a str, + normalized: RefCell>>, + fallback: RefCell<&'a mut TypeEquality<'a>>, +} + +impl Comparer<'_> { + fn equal(&self, left: &str, right: &str) -> Result { + match (self.normalized(left), self.normalized(right)) { + (Ok(left), Ok(right)) => Ok(left == right), + _ => (self.fallback.borrow_mut())(left, right), + } + } + + fn normalized(&self, type_sql: &str) -> Result { + let dialect: &str = self.dialect; + self.normalized + .borrow_mut() + .entry(type_sql.to_owned()) + .or_insert_with(|| normalize_type(type_sql, dialect).map(|found| found.normalized)) + .clone() + } +} diff --git a/crates/sqlbuild-rules/src/engine/main/build_models.rs b/crates/sqlbuild-rules/src/engine/main/build_models.rs new file mode 100644 index 0000000000..bf946ef825 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/build_models.rs @@ -0,0 +1,10 @@ +use crate::models::{Model, ModelRow, ModelRowsContext}; + +/// Build each model's rules facts and type proofs from its compiled-model row. +pub fn build_models( + rows: Vec, + context: ModelRowsContext<'_>, + fallback: impl FnMut(&str, &str) -> Result, +) -> Result, String> { + crate::engine::_helpers::rows::build_models(rows, context, fallback) +} diff --git a/crates/sqlbuild-rules/src/engine/main/evaluate.rs b/crates/sqlbuild-rules/src/engine/main/evaluate.rs deleted file mode 100644 index 9b038e4ae2..0000000000 --- a/crates/sqlbuild-rules/src/engine/main/evaluate.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub fn evaluate_json(request_json: &str) -> Result { - crate::engine::_helpers::evaluation::evaluate_json(request_json) -} diff --git a/crates/sqlbuild-rules/src/engine/main/evaluate_parsed.rs b/crates/sqlbuild-rules/src/engine/main/evaluate_parsed.rs deleted file mode 100644 index 81cc27c0fd..0000000000 --- a/crates/sqlbuild-rules/src/engine/main/evaluate_parsed.rs +++ /dev/null @@ -1,5 +0,0 @@ -use crate::models::EvaluateRequest; - -pub fn evaluate_parsed(request: EvaluateRequest) -> Result { - crate::engine::_helpers::evaluation::evaluate_request(request) -} diff --git a/crates/sqlbuild-rules/src/engine/main/evaluate_rows.rs b/crates/sqlbuild-rules/src/engine/main/evaluate_rows.rs new file mode 100644 index 0000000000..f61c3d771b --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/evaluate_rows.rs @@ -0,0 +1,7 @@ +use crate::errors::RowsError; +use crate::models::{RowsEvaluation, RowsRequest}; + +/// Answer a built request from the response memo or evaluate it, remembering the response. +pub fn evaluate_rows(request: RowsRequest) -> Result { + crate::engine::_helpers::memo::evaluate_rows(request) +} diff --git a/crates/sqlbuild-rules/src/engine/main/finalize.rs b/crates/sqlbuild-rules/src/engine/main/finalize.rs deleted file mode 100644 index 477f1980e8..0000000000 --- a/crates/sqlbuild-rules/src/engine/main/finalize.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub fn finalize_findings_json(request_json: &str) -> Result { - crate::engine::_helpers::evaluation::finalize_findings_json(request_json) -} diff --git a/crates/sqlbuild-rules/src/engine/main/finalize_rows.rs b/crates/sqlbuild-rules/src/engine/main/finalize_rows.rs new file mode 100644 index 0000000000..96e7839b0e --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/finalize_rows.rs @@ -0,0 +1,16 @@ +use crate::models::{Fault, RulesConfig}; + +/// Apply the exception policy once to completed built-in, SQL and custom findings. +pub fn finalize_rows( + project_dir: String, + config: RulesConfig, + evaluated_codes: &[String], + findings: Vec, +) -> Result, String> { + crate::engine::_helpers::evaluation::finalize_findings( + project_dir, + config, + evaluated_codes, + findings, + ) +} diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index d5349e927b..8cae1cda6c 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -1,5 +1,5 @@ +pub mod build_models; pub mod custom_host; -pub mod evaluate; -pub mod evaluate_parsed; -pub mod finalize; -pub mod parse_parts; +pub mod evaluate_rows; +pub mod finalize_rows; +pub mod project_model_rows; diff --git a/crates/sqlbuild-rules/src/engine/main/parse_parts.rs b/crates/sqlbuild-rules/src/engine/main/parse_parts.rs deleted file mode 100644 index 6b11c2b7b6..0000000000 --- a/crates/sqlbuild-rules/src/engine/main/parse_parts.rs +++ /dev/null @@ -1,9 +0,0 @@ -use crate::models::EvaluateRequest; - -pub fn parse_parts( - request_json: &[u8], - model_jsons: &[&[u8]], - model_digests: &[String], -) -> Result { - crate::engine::_helpers::evaluation::parse_parts(request_json, model_jsons, model_digests) -} diff --git a/crates/sqlbuild-rules/src/engine/main/project_model_rows.rs b/crates/sqlbuild-rules/src/engine/main/project_model_rows.rs new file mode 100644 index 0000000000..ab68082e76 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/project_model_rows.rs @@ -0,0 +1,19 @@ +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_analysis::compiled_project::models::CompiledProjectFacts; + +use crate::models::ModelRow; + +/// Rules rows for `names` in that order, read from the project's retained model facts. +pub fn project_model_rows( + project: &CompiledProjectFacts, + names: &[String], +) -> Result, String> { + names + .iter() + .map(|name| { + model_facts(project, name) + .ok_or_else(|| format!("compiled model '{name}' has no retained compile facts")) + .and_then(crate::engine::_helpers::project_rows::model_row) + }) + .collect() +} diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index 0203fa36cc..dd22e28f03 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -2,9 +2,9 @@ use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use tempfile::TempDir; -use crate::engine::main::evaluate::evaluate_json; -use crate::engine::main::evaluate_parsed::evaluate_parsed; -use crate::engine::main::parse_parts::parse_parts; +use crate::engine::_helpers::evaluation::evaluate_json; +use crate::engine::_helpers::evaluation::evaluate_request as evaluate_parsed; +use crate::engine::_helpers::evaluation::parse_parts; pub(crate) fn request(project_dir: &TempDir, config: &Value) -> String { json!({ diff --git a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs index 80dfd7d965..959acb40da 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs @@ -2,7 +2,7 @@ use serde_json::{Value, json}; use tempfile::TempDir; use crate::constants::NATIVE_BUILD_IDENTITY; -use crate::engine::main::evaluate::evaluate_json; +use crate::engine::_helpers::evaluation::evaluate_json; use crate::engine::tests::helpers; use crate::engine::tests::test_types; use crate::rules::_helpers::evaluation::normalize_rules_sql; diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index 8782307ec3..81853977f4 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use serde_json::{Value, json}; use tempfile::TempDir; -use crate::engine::main::parse_parts::parse_parts; +use crate::engine::_helpers::evaluation::parse_parts; use crate::engine::tests::helpers::{ cached_paths, evaluate_split, evaluate_whole, orders_models, split_request, uncached, write_model_files, diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_test_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_test_rules.rs index 6a6d2d26bb..ac3598f03d 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_test_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_test_rules.rs @@ -1,4 +1,4 @@ -use crate::engine::main::evaluate::evaluate_json; +use crate::engine::_helpers::evaluation::evaluate_json; use crate::engine::tests::{helpers, test_types}; use serde_json::{Value, json}; use tempfile::TempDir; diff --git a/crates/sqlbuild-rules/src/errors.rs b/crates/sqlbuild-rules/src/errors.rs new file mode 100644 index 0000000000..98cae0b073 --- /dev/null +++ b/crates/sqlbuild-rules/src/errors.rs @@ -0,0 +1,6 @@ +/// Why a natively built rules request produced no evaluation. +#[derive(Debug)] +pub enum RowsError { + Rules(String), + Io(std::io::Error), +} diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index ec95267231..690afcadde 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -5,8 +5,10 @@ pub mod configuration; pub mod constants; pub mod engine; +pub mod errors; pub mod models; pub mod rules; pub mod scope_metadata; pub mod sql_lint; pub mod sql_quality; +pub mod types; diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index e7576c976c..48117ae5f2 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -771,16 +771,6 @@ impl Default for EvaluateRequest { } } -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -pub(crate) struct FinalizeFindingsRequest { - pub version: u32, - pub project_dir: String, - pub config: RulesConfig, - pub evaluated_codes: Vec, - pub findings: Vec, -} - #[derive(Debug, Serialize)] pub(crate) struct EvaluateResponse { pub version: u32, @@ -865,3 +855,92 @@ pub struct CatalogueResponse { pub version: u32, pub rules: Vec, } + +/// One compiled model's facts as the rules request builder receives them. +#[derive(Clone, Debug, Default)] +pub struct ModelRow { + pub name: String, + pub relative_path: String, + pub query_sql: String, + pub authored_sql: String, + pub config: BTreeMap, + pub authored_config_keys: Vec, + pub logical_schema: Option, + pub references: Vec, + pub schema: Option, + /// Each inferred output column's name and type, in output order. + pub inferred_columns: Vec<(String, Option)>, + pub dynamic_proof: Option, + pub enum_columns: Vec, + pub enum_declarations: Vec, + pub constant_declarations: Vec, +} + +/// A model's declared schema entry. +#[derive(Clone, Debug, Default)] +pub struct SchemaRow { + /// Audits declared on the entry itself, not on its columns. + pub audit_count: u32, + pub columns: Vec, + pub dynamic_columns: Vec, +} + +#[derive(Clone, Debug, Default)] +pub struct SchemaColumnRow { + pub name: String, + pub data_type: Option, + pub nullable: Option, + pub audit_count: u32, +} + +/// One declared dynamic-column family; `key` is its name casefolded. +#[derive(Clone, Debug, Default)] +pub struct DynamicFamilyRow { + pub key: String, + pub name: String, + pub pivot_column: String, + pub value_column: String, + pub aggregate: String, + pub data_type: String, + pub name_pattern: Option, +} + +/// A model's dynamic-column contract proof; each family is keyed by its casefolded name. +#[derive(Clone, Debug, Default)] +pub struct DynamicProofRow { + pub output_proven: bool, + pub bare_dynamic_pivot: bool, + pub families: Vec<(String, Option)>, +} + +/// Project facts the model builder needs beyond each model's own row. +#[derive(Clone, Copy, Debug)] +pub struct ModelRowsContext<'a> { + pub sql_analysis_enabled: bool, + pub include_type_proof: bool, + pub dialect: &'a str, + /// The target model of every compiled audit attached to one. + pub attached_audit_targets: &'a [String], + pub sql_tests: &'a [SqlTestFact], +} + +/// A natively built rules request and the response memo it may be answered from. +#[derive(Debug)] +pub struct RowsRequest { + pub request: EvaluateRequest, + /// The compiled-code identity and memo path, when the rules cache is enabled. + pub memo: Option<(String, std::path::PathBuf)>, +} + +/// A rows request's decoded response. +#[derive(Debug, Deserialize)] +pub struct RowsEvaluation { + pub faults: Vec, + pub selected_codes: Vec, + pub evaluated_models: u64, + pub cache_hits: u64, + pub cache_misses: u64, + pub built_in_ms: u64, + #[serde(skip)] + pub reused: bool, +} diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/declared_keys.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/declared_keys.rs new file mode 100644 index 0000000000..cec3c2bdd1 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/declared_keys.rs @@ -0,0 +1,56 @@ +//! Unique keys one relation declares through its incremental config and column audits. + +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_core::text::models::PythonText; + +use crate::sql_quality::constants::{ + INCREMENTAL_MATERIALIZATION, KEY_PRESERVING_STRATEGIES, KEYED_DELETE_INSERT_STRATEGY, + MICROBATCH_MODE, NOT_NULL_AUDIT, UNIQUE_AUDIT, +}; +use crate::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; + +/// `unique_key` only where the incremental strategy keeps one row per key. +pub(crate) fn trusted_unique_key(relation: &DeclaredRelation) -> Vec> { + let strategy: String = relation.incremental_strategy.to_lowercase(); + let trusted: bool = relation.materialized.to_lowercase() == INCREMENTAL_MATERIALIZATION + && relation.incremental_mode.to_lowercase() != MICROBATCH_MODE + && KEY_PRESERVING_STRATEGIES.contains(&strategy.as_str()) + && !(strategy == KEYED_DELETE_INSERT_STRATEGY && relation.has_cursor); + match &relation.unique_key { + Some(key) if trusted && !key.is_empty() => vec![key.clone()], + _ => Vec::new(), + } +} + +/// Columns with a `unique` audit that are also non-null by audit or `nullable false`. +pub(crate) fn non_null_unique_columns( + python: PythonText, + relation: &DeclaredRelation, +) -> Vec { + let mut unique: Vec = Vec::new(); + let mut not_null: Vec = Vec::new(); + for column in &relation.columns { + if has_audit(column, UNIQUE_AUDIT) { + unique.push(column.name.clone()); + } + if has_audit(column, NOT_NULL_AUDIT) || column.not_nullable { + not_null.push(python_casefold(python, &column.name)); + } + } + for (definition, argument) in &relation.audits { + let Some(argument) = argument else { + continue; + }; + if definition == NOT_NULL_AUDIT { + not_null.push(python_casefold(python, argument)); + } else if definition == UNIQUE_AUDIT { + unique.push(argument.clone()); + } + } + unique.retain(|column| not_null.contains(&python_casefold(python, column))); + unique +} + +fn has_audit(column: &DeclaredKeyColumn, name: &str) -> bool { + column.audits.iter().any(|audit| audit == name) +} diff --git a/crates/sqlbuild-rules/src/sql_quality/constants.rs b/crates/sqlbuild-rules/src/sql_quality/constants.rs index cb570ec0f7..fbf0f8a3f3 100644 --- a/crates/sqlbuild-rules/src/sql_quality/constants.rs +++ b/crates/sqlbuild-rules/src/sql_quality/constants.rs @@ -191,3 +191,12 @@ pub(crate) const INTERPOLATION_SENTINEL_PREFIXES: [&str; 3] = [ "__sqlbuild_context_parameter_", ]; pub(crate) const FIRST_ROW: &str = "1"; + +pub(crate) const UNIQUE_AUDIT: &str = "unique"; +pub(crate) const NOT_NULL_AUDIT: &str = "not_null"; +pub(crate) const INCREMENTAL_MATERIALIZATION: &str = "incremental"; +pub(crate) const MICROBATCH_MODE: &str = "microbatch"; +pub(crate) const KEYED_DELETE_INSERT_STRATEGY: &str = "delete_insert"; +pub(crate) const KEY_PRESERVING_STRATEGIES: [&str; 2] = ["merge", KEYED_DELETE_INSERT_STRATEGY]; +pub(crate) const RELATION_KIND_REF: &str = "ref"; +pub(crate) const RELATION_KIND_SEED: &str = "seed"; diff --git a/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs b/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs new file mode 100644 index 0000000000..ed06cbb788 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs @@ -0,0 +1,39 @@ +use std::collections::BTreeMap; + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::models::PythonText; + +use crate::sql_quality::constants::{RELATION_KIND_REF, RELATION_KIND_SEED}; +use crate::sql_quality::declared_keys::{non_null_unique_columns, trusted_unique_key}; +use crate::sql_quality::models::DeclaredRelation; + +/// Keys that prove one row per key, by `kind:name` identity; seeds are also keyed as refs. +pub fn declared_relation_keys( + relations: &[DeclaredRelation], +) -> BTreeMap>> { + let python: PythonText = active_python_text(); + let mut keys: BTreeMap>> = BTreeMap::new(); + for relation in relations { + let mut found: Vec> = trusted_unique_key(relation); + for column in non_null_unique_columns(python, relation) { + found.push(vec![column]); + } + if found.is_empty() { + continue; + } + let mut kinds: Vec<&str> = vec![relation.kind.as_str()]; + if relation.kind == RELATION_KIND_SEED { + kinds.push(RELATION_KIND_REF); + } + for kind in kinds { + let entry: &mut Vec> = + keys.entry(format!("{kind}:{}", relation.name)).or_default(); + for key in &found { + if !entry.contains(key) { + entry.push(key.clone()); + } + } + } + } + keys +} diff --git a/crates/sqlbuild-rules/src/sql_quality/main/mod.rs b/crates/sqlbuild-rules/src/sql_quality/main/mod.rs index 4c630487f1..f8c958d3fb 100644 --- a/crates/sqlbuild-rules/src/sql_quality/main/mod.rs +++ b/crates/sqlbuild-rules/src/sql_quality/main/mod.rs @@ -1 +1,2 @@ +pub mod declared_relation_keys; pub mod diagnostics; diff --git a/crates/sqlbuild-rules/src/sql_quality/mod.rs b/crates/sqlbuild-rules/src/sql_quality/mod.rs index 520f7e2f3c..2c9b78b9fc 100644 --- a/crates/sqlbuild-rules/src/sql_quality/mod.rs +++ b/crates/sqlbuild-rules/src/sql_quality/mod.rs @@ -3,6 +3,8 @@ pub mod constants; #[path = "_helpers/cte_columns.rs"] mod cte_columns; +#[path = "_helpers/declared_keys.rs"] +mod declared_keys; #[path = "_helpers/json_parses.rs"] mod json_parses; #[path = "_helpers/keys.rs"] diff --git a/crates/sqlbuild-rules/src/sql_quality/models.rs b/crates/sqlbuild-rules/src/sql_quality/models.rs index 30d00e522f..d57f1aee24 100644 --- a/crates/sqlbuild-rules/src/sql_quality/models.rs +++ b/crates/sqlbuild-rules/src/sql_quality/models.rs @@ -16,3 +16,28 @@ pub(crate) struct QualityRequest<'a> { pub(crate) externally_referenced_ctes: &'a HashSet, pub(crate) fixture_body: bool, } + +/// One project relation's declared key facts: its incremental config and audited columns. +#[derive(Debug, Clone, Default)] +pub struct DeclaredRelation { + /// `ref`, `seed` or `source`. + pub kind: String, + pub name: String, + pub materialized: String, + pub incremental_mode: String, + pub incremental_strategy: String, + pub has_cursor: bool, + /// The configured `unique_key` columns, when it is a name or a list of names. + pub unique_key: Option>, + pub columns: Vec, + /// Table-level audits as `(definition name, its string column argument)`. + pub audits: Vec<(String, Option)>, +} + +/// One declared column: its name, its audit definition names and whether it is declared non-null. +#[derive(Debug, Clone, Default)] +pub struct DeclaredKeyColumn { + pub name: String, + pub audits: Vec, + pub not_nullable: bool, +} diff --git a/crates/sqlbuild-rules/src/types.rs b/crates/sqlbuild-rules/src/types.rs new file mode 100644 index 0000000000..90e92f3f5a --- /dev/null +++ b/crates/sqlbuild-rules/src/types.rs @@ -0,0 +1,2 @@ +/// Answers type equality the native type system cannot normalize. +pub type TypeEquality<'a> = dyn FnMut(&str, &str) -> Result + 'a; diff --git a/crates/sqlbuild-scopes/Cargo.toml b/crates/sqlbuild-scopes/Cargo.toml index 5d11915923..9ecd0c9934 100644 --- a/crates/sqlbuild-scopes/Cargo.toml +++ b/crates/sqlbuild-scopes/Cargo.toml @@ -8,7 +8,12 @@ publish.workspace = true rust-version.workspace = true [dependencies] +serde_json.workspace = true +sha2.workspace = true +sqlbuild-config.workspace = true sqlbuild-core.workspace = true +sqlbuild-discovery.workspace = true +sqlbuild-render.workspace = true sqlbuild-sqltext.workspace = true [lints] diff --git a/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs b/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs new file mode 100644 index 0000000000..04abf7838f --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod schema_checks; diff --git a/crates/sqlbuild-scopes/src/compile_usages/_helpers/schema_checks.rs b/crates/sqlbuild-scopes/src/compile_usages/_helpers/schema_checks.rs new file mode 100644 index 0000000000..b5797c52a0 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/_helpers/schema_checks.rs @@ -0,0 +1,105 @@ +//! Whether a schema can see the schema it extends and its enum column types, and the S006s. + +use crate::compile_usages::constants::{ + CURRENT_FOLDER, DECLARATION_STEM, INACCESSIBLE_CODE, SCHEMA_KIND, +}; +use crate::compile_usages::models::{InaccessibleDiagnostic, NamedRecord, RejectedScopePath}; +use crate::scope_index::main::path_visible::path_visible; + +/// `declaration_lexical_path`: the declaration file path whose folder its owner controls. +pub(crate) fn lexical_path(record: &NamedRecord) -> String { + format!("{}/{DECLARATION_STEM}", folder(record)) +} + +fn folder(record: &NamedRecord) -> &str { + record + .declared + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(CURRENT_FOLDER) +} + +/// The owner a message names: the owning folder, else the ownership root. +fn owner(record: &NamedRecord) -> &str { + record + .declared + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(&record.ownership_root) +} + +/// Whether `declaration` is visible from the schema `consumer`'s declaration file. +pub(crate) fn visible_from( + declaration: &NamedRecord, + consumer: &NamedRecord, +) -> Result { + let path: String = lexical_path(consumer); + path_visible(&declaration.declared, &path).map_err(|_| RejectedScopePath { + declared: declaration.declared.clone(), + path, + }) +} + +/// The S006 of a schema extending a schema its folder cannot see. +pub(crate) fn hidden_parent( + consumer: &NamedRecord, + name: &str, + parent: &NamedRecord, +) -> InaccessibleDiagnostic { + InaccessibleDiagnostic { + key: vec![ + INACCESSIBLE_CODE.to_owned(), + consumer.path.clone(), + SCHEMA_KIND.to_owned(), + name.to_owned(), + ], + message: format!( + "Schema '{name}' is not visible from '{}'. It is defined at {} with {} scope owned \ + by '{}'", + consumer.path, + parent.path, + parent.declared.scope.as_str(), + owner(parent) + ), + path: consumer.path.clone(), + help: format!( + "move '{}' to a schemas/ role owned by '{}' or one of its parent folders, or to the \ + project-wide schemas/ when consumers span resource trees", + parent.path, + folder(consumer) + ), + } +} + +/// The S006 of a schema column typed with an enum the schema cannot see. +pub(crate) fn hidden_enum( + schema: (&NamedRecord, &str), + column: (&str, &str), + enum_record: &NamedRecord, +) -> InaccessibleDiagnostic { + let (record, schema_name) = schema; + let (column_name, enum_name) = column; + InaccessibleDiagnostic { + key: vec![ + INACCESSIBLE_CODE.to_owned(), + record.path.clone(), + schema_name.to_owned(), + column_name.to_owned(), + ], + message: format!( + "Schema '{schema_name}' in {} column '{column_name}' uses enum '{enum_name}', which \ + is not visible from the schema's location. The enum is defined at {} with {} scope \ + owned by '{}'", + record.path, + enum_record.path, + enum_record.declared.scope.as_str(), + owner(enum_record) + ), + path: record.path.clone(), + help: "move the enum to a role visible from the schema's owning folder, or move the \ + schema next to the enum" + .to_owned(), + } +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/constants.rs b/crates/sqlbuild-scopes/src/compile_usages/constants.rs new file mode 100644 index 0000000000..519de89375 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/constants.rs @@ -0,0 +1,7 @@ +//! The inaccessible-declaration code and the declaration file stem schema visibility reads. + +pub(crate) const INACCESSIBLE_CODE: &str = "S006"; +pub(crate) const SCHEMA_KIND: &str = "schema"; +/// The authored path whose folder controls what a declaration file can use. +pub(crate) const DECLARATION_STEM: &str = "__declaration__.sql"; +pub(crate) const CURRENT_FOLDER: &str = "."; diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/distinct_positions.rs b/crates/sqlbuild-scopes/src/compile_usages/main/distinct_positions.rs new file mode 100644 index 0000000000..b1a563fe9d --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/main/distinct_positions.rs @@ -0,0 +1,17 @@ +//! `dict.fromkeys` over keys: the position of each key's first occurrence, in order. + +use std::collections::HashSet; +use std::hash::Hash; + +/// The positions of the first occurrence of each distinct key. +#[must_use] +pub fn distinct_positions(keys: &[K]) -> Vec { + let mut seen: HashSet<&K> = HashSet::with_capacity(keys.len()); + let mut positions: Vec = Vec::new(); + for (position, key) in keys.iter().enumerate() { + if seen.insert(key) { + positions.push(position); + } + } + positions +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/merged_dependencies.rs b/crates/sqlbuild-scopes/src/compile_usages/main/merged_dependencies.rs new file mode 100644 index 0000000000..7aa9399940 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/main/merged_dependencies.rs @@ -0,0 +1,41 @@ +//! A macro declaration's dependencies merged with the declarations its runtime uses consumed. + +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; + +use crate::compile_usages::models::DependencySource; + +/// Per declaration, `None` without macro metadata, else own then consumed dependencies, deduped. +#[must_use] +pub fn merged_dependencies( + declarations: &[(K, Option>)], + usages: &[(Option, K)], +) -> Vec>> { + let mut by_consumer: HashMap<&K, Vec> = HashMap::new(); + for (position, (consumer, _)) in usages.iter().enumerate() { + if let Some(consumer) = consumer { + by_consumer.entry(consumer).or_default().push(position); + } + } + let mut merged: Vec>> = Vec::with_capacity(declarations.len()); + for (identity, own) in declarations { + let Some(own) = own else { + merged.push(None); + continue; + }; + let mut seen: HashSet<&K> = HashSet::new(); + let mut sources: Vec = Vec::new(); + for (position, dependency) in own.iter().enumerate() { + if seen.insert(dependency) { + sources.push(DependencySource::Own(position)); + } + } + for position in by_consumer.get(identity).into_iter().flatten() { + if seen.insert(&usages[*position].1) { + sources.push(DependencySource::Usage(*position)); + } + } + merged.push(Some(sources)); + } + merged +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs b/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs new file mode 100644 index 0000000000..b8f3347ba3 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/main/mod.rs @@ -0,0 +1,3 @@ +pub mod distinct_positions; +pub mod merged_dependencies; +pub mod schema_scope_usages; diff --git a/crates/sqlbuild-scopes/src/compile_usages/main/schema_scope_usages.rs b/crates/sqlbuild-scopes/src/compile_usages/main/schema_scope_usages.rs new file mode 100644 index 0000000000..cb2ae39454 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/main/schema_scope_usages.rs @@ -0,0 +1,55 @@ +//! `model_schema_scope_usages`: parent schemas and enum column types each schema uses. + +use std::collections::{HashMap, HashSet}; + +use crate::compile_usages::_helpers::schema_checks::{hidden_enum, hidden_parent, visible_from}; +use crate::compile_usages::models::{ + NamedRecord, RejectedScopePath, SchemaDeclaration, SchemaUsages, +}; + +/// Each indexed schema's usages of its parent and enum column types, with S006 where hidden. +pub fn schema_scope_usages( + schemas: &[SchemaDeclaration], + schema_records: &HashMap, + enum_records: &HashMap, +) -> Result { + let mut scan: SchemaUsages = SchemaUsages::default(); + let mut seen: HashSet<(usize, usize)> = HashSet::new(); + for schema in schemas { + let Some(record) = schema_records.get(&schema.name) else { + continue; + }; + let mut found: Vec = Vec::new(); + if let Some(parent_name) = &schema.extends + && let Some(parent) = schema_records.get(parent_name) + { + if !visible_from(parent, record)? { + scan.diagnostics + .push(hidden_parent(record, parent_name, parent)); + } + found.push(parent.handle); + } + for (column, column_type) in &schema.columns { + let Some(enum_name) = column_type else { + continue; + }; + let Some(enum_record) = enum_records.get(enum_name) else { + continue; + }; + if !visible_from(enum_record, record)? { + scan.diagnostics.push(hidden_enum( + (record, &schema.name), + (column, enum_name), + enum_record, + )); + } + found.push(enum_record.handle); + } + for declaration in found { + if seen.insert((record.handle, declaration)) { + scan.usages.push((record.handle, declaration)); + } + } + } + Ok(scan) +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/mod.rs b/crates/sqlbuild-scopes/src/compile_usages/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-scopes/src/compile_usages/models.rs b/crates/sqlbuild-scopes/src/compile_usages/models.rs new file mode 100644 index 0000000000..1ed254abf0 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/models.rs @@ -0,0 +1,53 @@ +//! The schema declarations and indexed records compile usage facts are read from. + +use crate::scope_index::models::DeclaredScope; + +/// The first indexed record of a named schema or enum, by its caller handle. +#[derive(Clone, Debug)] +pub struct NamedRecord { + /// The caller's handle for this record, echoed in usages. + pub handle: usize, + pub path: String, + pub declared: DeclaredScope, + pub ownership_root: String, +} + +/// One reusable schema declaration: its name, the schema it extends, and each column's type. +#[derive(Clone, Debug)] +pub struct SchemaDeclaration { + pub name: String, + pub extends: Option, + pub columns: Vec<(String, Option)>, +} + +/// An S006 diagnostic: its report key, message, path and help. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InaccessibleDiagnostic { + pub key: Vec, + pub message: String, + pub path: String, + pub help: String, +} + +/// Each distinct `(consumer handle, declaration handle)` usage and the S006s found on the way. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SchemaUsages { + pub usages: Vec<(usize, usize)>, + pub diagnostics: Vec, +} + +/// A path Python's visibility check rejects, with the scope it was checked against. +#[derive(Clone, Debug)] +pub struct RejectedScopePath { + pub declared: DeclaredScope, + pub path: String, +} + +/// Where one merged macro dependency comes from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DependencySource { + /// The declaration's own dependency at this position. + Own(usize), + /// The declaration of the collected usage at this position. + Usage(usize), +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/tests.rs b/crates/sqlbuild-scopes/src/compile_usages/tests.rs new file mode 100644 index 0000000000..ff4195822a --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_compile_usages.rs"] +mod compile_usages; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-scopes/src/compile_usages/tests/helpers.rs b/crates/sqlbuild-scopes/src/compile_usages/tests/helpers.rs new file mode 100644 index 0000000000..21aa7700ca --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/tests/helpers.rs @@ -0,0 +1,96 @@ +use std::collections::HashMap; + +use crate::compile_usages::models::{NamedRecord, SchemaDeclaration}; +use crate::scope_index::models::{DeclaredScope, ScopeKind}; + +fn record(handle: usize, path: &str, scope: ScopeKind, owning_path: Option<&str>) -> NamedRecord { + NamedRecord { + handle, + path: path.to_owned(), + declared: DeclaredScope { + scope, + owner: None, + owning_path: owning_path.map(str::to_owned), + }, + ownership_root: "models".to_owned(), + } +} + +pub(super) fn sales_schemas() -> Vec { + vec![ + SchemaDeclaration { + name: "orders_schema".to_owned(), + extends: Some("base_schema".to_owned()), + columns: vec![ + ("status".to_owned(), Some("order_status".to_owned())), + ("amount".to_owned(), Some("DECIMAL".to_owned())), + ("note".to_owned(), None), + ], + }, + SchemaDeclaration { + name: "unindexed".to_owned(), + extends: None, + columns: Vec::new(), + }, + ] +} + +pub(super) fn visible_schema_records() -> HashMap { + HashMap::from([ + ( + "orders_schema".to_owned(), + record( + 0, + "models/sales/schemas/orders.yml", + ScopeKind::Inherited, + Some("models/sales"), + ), + ), + ( + "base_schema".to_owned(), + record(1, "schemas/base.yml", ScopeKind::Global, None), + ), + ]) +} + +pub(super) fn hidden_schema_records() -> HashMap { + HashMap::from([ + ( + "orders_schema".to_owned(), + record( + 0, + "models/sales/schemas/orders.yml", + ScopeKind::Inherited, + Some("models/sales"), + ), + ), + ( + "base_schema".to_owned(), + record( + 1, + "models/finance/schemas/base.yml", + ScopeKind::Inherited, + Some("models/finance"), + ), + ), + ]) +} + +pub(super) fn visible_enum_records() -> HashMap { + HashMap::from([( + "order_status".to_owned(), + record(2, "enums/status.yml", ScopeKind::Global, None), + )]) +} + +pub(super) fn hidden_enum_records() -> HashMap { + HashMap::from([( + "order_status".to_owned(), + record( + 2, + "models/finance/enums/status.yml", + ScopeKind::Local, + Some("models/finance"), + ), + )]) +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs b/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs new file mode 100644 index 0000000000..2ad12b9697 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/tests/test_compile_usages.rs @@ -0,0 +1,117 @@ +use crate::compile_usages::main::distinct_positions::distinct_positions; +use crate::compile_usages::main::merged_dependencies::merged_dependencies; +use crate::compile_usages::main::schema_scope_usages::schema_scope_usages; +use crate::compile_usages::models::{DependencySource, SchemaUsages}; +use crate::compile_usages::tests::helpers::{ + hidden_enum_records, hidden_schema_records, sales_schemas, visible_enum_records, + visible_schema_records, +}; +use crate::compile_usages::tests::test_types::{ + DistinctPositionsTestCase, MergedDependenciesTestCase, SchemaScopeUsagesTestCase, +}; + +#[test] +fn given_schemas_when_collecting_scope_usages_then_python_usages_and_s006_hold() { + let test_cases = [ + SchemaScopeUsagesTestCase { + description: "a visible parent and enum are used without diagnostics", + schemas: sales_schemas, + schema_records: visible_schema_records, + enum_records: visible_enum_records, + expected_usages: &[(0, 1), (0, 2)], + expected_messages: &[], + }, + SchemaScopeUsagesTestCase { + description: "a hidden parent and enum are still used and each reports S006", + schemas: sales_schemas, + schema_records: hidden_schema_records, + enum_records: hidden_enum_records, + expected_usages: &[(0, 1), (0, 2)], + expected_messages: &[ + "Schema 'base_schema' is not visible from 'models/sales/schemas/orders.yml'. It is defined at models/finance/schemas/base.yml with inherited scope owned by 'models/finance'", + "Schema 'orders_schema' in models/sales/schemas/orders.yml column 'status' uses enum 'order_status', which is not visible from the schema's location. The enum is defined at models/finance/enums/status.yml with local scope owned by 'models/finance'", + ], + }, + ]; + for test_case in test_cases { + let scan: SchemaUsages = schema_scope_usages( + &(test_case.schemas)(), + &(test_case.schema_records)(), + &(test_case.enum_records)(), + ) + .expect("valid paths"); + + let messages = scan + .diagnostics + .iter() + .map(|diagnostic| diagnostic.message.as_str()) + .collect::>(); + assert_eq!( + (scan.usages.as_slice(), messages.as_slice()), + (test_case.expected_usages, test_case.expected_messages), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_declarations_when_merging_usages_then_own_then_consumed_dependencies_dedupe() { + let test_cases = [MergedDependenciesTestCase { + description: "own dependencies first, then each consumed declaration once", + declarations: &[ + ("format_amount", Some(&["round_money", "round_money"])), + ("order_status", None), + ("helper", Some(&[])), + ], + usages: &[ + (Some("format_amount"), "currency"), + (None, "format_amount"), + (Some("format_amount"), "round_money"), + (Some("format_amount"), "currency"), + ], + expected_sources: &[ + Some(&[DependencySource::Own(0), DependencySource::Usage(0)]), + None, + Some(&[]), + ], + }]; + for test_case in test_cases { + let declarations = test_case + .declarations + .iter() + .map(|(name, own)| (*name, own.map(<[&str]>::to_vec))) + .collect::>(); + + let merged = merged_dependencies(&declarations, test_case.usages); + + assert_eq!( + merged, + test_case + .expected_sources + .iter() + .map(|sources| sources.map(<[DependencySource]>::to_vec)) + .collect::>(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_keys_when_taking_distinct_positions_then_first_occurrences_keep_order() { + let test_cases = [DistinctPositionsTestCase { + description: "repeats keep their first position", + keys: &["orders", "customers", "orders", "products", "customers"], + expected_positions: &[0, 1, 3], + }]; + for test_case in test_cases { + let positions: Vec = distinct_positions(test_case.keys); + + assert_eq!( + positions, test_case.expected_positions, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/compile_usages/tests/test_types.rs b/crates/sqlbuild-scopes/src/compile_usages/tests/test_types.rs new file mode 100644 index 0000000000..0556e26c02 --- /dev/null +++ b/crates/sqlbuild-scopes/src/compile_usages/tests/test_types.rs @@ -0,0 +1,27 @@ +use crate::compile_usages::models::{DependencySource, NamedRecord, SchemaDeclaration}; +use std::collections::HashMap; + +/// Schemas, their indexed records, and the usages and S006 messages they yield. +pub(super) struct SchemaScopeUsagesTestCase { + pub(super) description: &'static str, + pub(super) schemas: fn() -> Vec, + pub(super) schema_records: fn() -> HashMap, + pub(super) enum_records: fn() -> HashMap, + pub(super) expected_usages: &'static [(usize, usize)], + pub(super) expected_messages: &'static [&'static str], +} + +/// Declarations with their own dependencies, collected usages, and the merged sources. +pub(super) struct MergedDependenciesTestCase { + pub(super) description: &'static str, + pub(super) declarations: &'static [(&'static str, Option<&'static [&'static str]>)], + pub(super) usages: &'static [(Option<&'static str>, &'static str)], + pub(super) expected_sources: &'static [Option<&'static [DependencySource]>], +} + +/// Keys and the positions of their first occurrences. +pub(super) struct DistinctPositionsTestCase { + pub(super) description: &'static str, + pub(super) keys: &'static [&'static str], + pub(super) expected_positions: &'static [usize], +} diff --git a/crates/sqlbuild-scopes/src/lib.rs b/crates/sqlbuild-scopes/src/lib.rs index ea218f103e..8fdcac1590 100644 --- a/crates/sqlbuild-scopes/src/lib.rs +++ b/crates/sqlbuild-scopes/src/lib.rs @@ -2,5 +2,9 @@ #![forbid(unsafe_code)] +pub mod compile_usages; +pub mod placement; pub mod relationship_names; +pub mod scope_cache; pub mod scope_index; +pub mod scope_report; diff --git a/crates/sqlbuild-scopes/src/placement/_helpers/anchors.rs b/crates/sqlbuild-scopes/src/placement/_helpers/anchors.rs new file mode 100644 index 0000000000..bdce356fcc --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/_helpers/anchors.rs @@ -0,0 +1,142 @@ +//! The resource folders each declaration's consumers anchor it to, through consumer chains. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use crate::placement::_helpers::roles::is_named_kind; +use crate::placement::constants::PROJECT_ROOT_PATH; +use crate::placement::models::{PlacementIndex, PlacementUsage}; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, OwnershipRoot, OwnershipRootKind, QualifiedIdentity, + ResourceIdentity, ResourceRecord, +}; +use sqlbuild_core::posix_path::main::posix_parent::posix_parent; + +/// An ownership root and the folder below it a consumer sits in. +pub(crate) type Anchor = (OwnershipRoot, String); +/// Each declaration's usages, in index order. +pub(crate) type UsagesByDeclaration<'a> = HashMap<&'a DeclarationIdentity, Vec<&'a PlacementUsage>>; +/// Each declaration's anchors. +pub(crate) type AnchorSets = HashMap>; + +/// `_usages_by_declaration`. +pub(crate) fn usages_by_declaration(index: &PlacementIndex) -> UsagesByDeclaration<'_> { + let mut grouped: UsagesByDeclaration<'_> = HashMap::new(); + for usage in &index.usages { + grouped.entry(&usage.declaration).or_default().push(usage); + } + grouped +} + +/// `_anchor_sets`: direct anchors, then anchors propagated to declarations their dependents use. +pub(crate) fn anchor_sets(index: &PlacementIndex, usages: &UsagesByDeclaration<'_>) -> AnchorSets { + let resources: HashMap<&ResourceIdentity, &ResourceRecord> = index + .resources + .iter() + .map(|record| (&record.identity, record)) + .collect(); + let declarations: HashMap<&DeclarationIdentity, &DeclarationRecord> = index + .declarations + .iter() + .map(|record| (&record.identity, record)) + .collect(); + let mut anchors: AnchorSets = HashMap::new(); + let mut dependents: HashMap> = HashMap::new(); + for (identity, declaration_usages) in usages { + let mut direct: HashSet = HashSet::new(); + for usage in declaration_usages { + match direct_anchor(usage, &resources, &declarations) { + DirectAnchor::Anchor(anchor) => { + direct.insert(anchor); + } + DirectAnchor::Dependent(consumer) => { + dependents + .entry(consumer.clone()) + .or_default() + .insert((*identity).clone()); + } + DirectAnchor::None => {} + } + } + anchors + .entry((*identity).clone()) + .or_default() + .extend(direct); + } + let mut pending: VecDeque = anchors.keys().cloned().collect(); + while let Some(source) = pending.pop_front() { + let source_anchors: HashSet = anchors.get(&source).cloned().unwrap_or_default(); + let Some(targets) = dependents.get(&source) else { + continue; + }; + for target in targets { + let target_anchors: &mut HashSet = anchors.entry(target.clone()).or_default(); + if source_anchors.is_subset(target_anchors) { + continue; + } + target_anchors.extend(source_anchors.iter().cloned()); + pending.push_back(target.clone()); + } + } + anchors +} + +/// What one usage contributes: an anchor, a declaration consumer to inherit from, or nothing. +enum DirectAnchor<'a> { + Anchor(Anchor), + Dependent(&'a DeclarationIdentity), + None, +} + +fn direct_anchor<'a>( + usage: &'a PlacementUsage, + resources: &HashMap<&ResourceIdentity, &ResourceRecord>, + declarations: &HashMap<&DeclarationIdentity, &DeclarationRecord>, +) -> DirectAnchor<'a> { + if let Some(QualifiedIdentity::Declaration(through)) = &usage.through { + return match declarations.get(through) { + Some(record) => match &record.owning_path { + Some(path) => DirectAnchor::Anchor((record.ownership_root.clone(), path.clone())), + None => DirectAnchor::None, + }, + None => DirectAnchor::None, + }; + } + if let (None, QualifiedIdentity::Declaration(consumer)) = (&usage.through, &usage.consumer) + && is_named_kind(consumer.kind) + { + return match declarations.get(consumer) { + Some(record) => DirectAnchor::Anchor(location_anchor(record)), + None => DirectAnchor::None, + }; + } + let resource: Option<&ResourceIdentity> = match (&usage.through, &usage.consumer) { + (Some(QualifiedIdentity::Resource(through)), _) => Some(through), + (None, QualifiedIdentity::Resource(consumer)) => Some(consumer), + _ => None, + }; + match (resource, &usage.consumer) { + (Some(identity), _) => match resources.get(identity) { + Some(record) => { + DirectAnchor::Anchor((record.ownership_root.clone(), posix_parent(&record.path))) + } + None => DirectAnchor::None, + }, + (None, QualifiedIdentity::Declaration(consumer)) => DirectAnchor::Dependent(consumer), + (None, QualifiedIdentity::Resource(_)) => DirectAnchor::None, + } +} + +/// `_declaration_location_anchor`: where an audit, schema or hook using a declaration lives. +fn location_anchor(record: &DeclarationRecord) -> Anchor { + match &record.owning_path { + None => ( + OwnershipRoot { + path: PROJECT_ROOT_PATH.to_owned(), + kind: OwnershipRootKind::Global, + resource_kind: None, + }, + PROJECT_ROOT_PATH.to_owned(), + ), + Some(path) => (record.ownership_root.clone(), path.clone()), + } +} diff --git a/crates/sqlbuild-scopes/src/placement/_helpers/mod.rs b/crates/sqlbuild-scopes/src/placement/_helpers/mod.rs new file mode 100644 index 0000000000..9799141027 --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod anchors; +pub(crate) mod posix_paths; +pub(crate) mod required; +pub(crate) mod roles; diff --git a/crates/sqlbuild-scopes/src/placement/_helpers/posix_paths.rs b/crates/sqlbuild-scopes/src/placement/_helpers/posix_paths.rs new file mode 100644 index 0000000000..6c43e6204a --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/_helpers/posix_paths.rs @@ -0,0 +1,19 @@ +//! The common prefix of `PurePosixPath` parts, as `_lca` joins it. + +use sqlbuild_core::posix_path::main::posix_join::posix_join; +use sqlbuild_core::posix_path::main::posix_parts::posix_parts; + +/// The longest common leading parts of `paths`, as `_lca` joins them. +pub(crate) fn common_prefix(paths: &[&str]) -> String { + let split: Vec> = paths.iter().map(|path| posix_parts(path)).collect(); + let shortest: usize = split.iter().map(Vec::len).min().unwrap_or(0); + let mut common: Vec<&str> = Vec::new(); + for position in 0..shortest { + let component: &str = split[0][position]; + if split.iter().any(|item| item[position] != component) { + break; + } + common.push(component); + } + posix_join(&common) +} diff --git a/crates/sqlbuild-scopes/src/placement/_helpers/required.rs b/crates/sqlbuild-scopes/src/placement/_helpers/required.rs new file mode 100644 index 0000000000..d4cf8e94e7 --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/_helpers/required.rs @@ -0,0 +1,112 @@ +//! `_required_placement_for_record`: the scope and folder a declaration's consumers require. + +use std::collections::{BTreeSet, HashSet}; + +use crate::placement::_helpers::anchors::{Anchor, AnchorSets, UsagesByDeclaration}; +use crate::placement::_helpers::posix_paths::common_prefix; +use crate::placement::_helpers::roles::is_named_kind; +use crate::placement::models::{PlacementUsage, RequiredPlacement}; +use crate::scope_index::main::format_identity::format_identity; +use crate::scope_index::models::{ + DeclarationKind, DeclarationRecord, OwnershipRoot, OwnershipRootKind, QualifiedIdentity, + ScopeKind, +}; + +/// Whether a private declaration's owner uses it directly. +pub(crate) fn owner_uses_directly(record: &DeclarationRecord, usage: &PlacementUsage) -> bool { + let owner: Option = record + .identity + .owner + .clone() + .map(QualifiedIdentity::Resource); + usage.through.is_none() && owner.as_ref() == Some(&usage.consumer) +} + +/// The placement `record`'s consumers require, or `None` when it has no anchored consumers. +pub(crate) fn required_placement( + record: &DeclarationRecord, + usages: &UsagesByDeclaration<'_>, + anchor_sets: &AnchorSets, +) -> Option { + let declaration_usages: &[&PlacementUsage] = usages + .get(&record.identity) + .map(Vec::as_slice) + .unwrap_or_default(); + if record.scope == ScopeKind::Private { + let direct: bool = declaration_usages + .iter() + .any(|usage| owner_uses_directly(record, usage)); + let path: String = record + .owning_path + .clone() + .filter(|path| !path.is_empty()) + .unwrap_or_else(|| record.ownership_root.path.clone()); + return direct.then_some(RequiredPlacement { + scope: ScopeKind::Private, + path: Some(path), + consumers: Vec::new(), + }); + } + if declaration_usages.is_empty() { + return None; + } + let anchors: &HashSet = anchor_sets.get(&record.identity)?; + if anchors.is_empty() { + return None; + } + let consumers: Vec = declaration_usages + .iter() + .map(|usage| consumer_label(usage)) + .collect::>() + .into_iter() + .collect(); + let global = |consumers: Vec| RequiredPlacement { + scope: ScopeKind::Global, + path: None, + consumers, + }; + let roots: HashSet<&OwnershipRoot> = anchors.iter().map(|(root, _)| root).collect(); + let single_root: Option<&OwnershipRoot> = match roots.iter().next() { + Some(root) if roots.len() == 1 && root.kind != OwnershipRootKind::Global => Some(root), + _ => None, + }; + let Some(ownership_root) = single_root else { + return Some(global(consumers)); + }; + let distinct: HashSet<&str> = anchors.iter().map(|(_, path)| path.as_str()).collect(); + if let [required_path] = distinct.iter().copied().collect::>().as_slice() { + if is_named_kind(record.identity.kind) && *required_path == ownership_root.path { + return Some(global(consumers)); + } + let scope: ScopeKind = if record.identity.kind == DeclarationKind::SingularAudit { + ScopeKind::Inherited + } else { + ScopeKind::Local + }; + return Some(RequiredPlacement { + scope, + path: Some((*required_path).to_owned()), + consumers, + }); + } + let paths: Vec<&str> = anchors.iter().map(|(_, path)| path.as_str()).collect(); + let required_path: String = common_prefix(&paths); + if required_path == ownership_root.path { + return Some(global(consumers)); + } + Some(RequiredPlacement { + scope: ScopeKind::Inherited, + path: Some(required_path), + consumers, + }) +} + +/// `_consumer_label`: the consumer, and what it uses the declaration through. +fn consumer_label(usage: &PlacementUsage) -> String { + let mut label: String = format_identity(&usage.consumer); + if let Some(through) = &usage.through { + label.push_str(" through "); + label.push_str(&format_identity(through)); + } + label +} diff --git a/crates/sqlbuild-scopes/src/placement/_helpers/roles.rs b/crates/sqlbuild-scopes/src/placement/_helpers/roles.rs new file mode 100644 index 0000000000..6021f9e60e --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/_helpers/roles.rs @@ -0,0 +1,21 @@ +//! Declaration roles, named kinds and the scope labels placement messages print. + +use crate::scope_index::models::{DeclarationKind, ScopeKind}; + +/// Kinds declared in named roles, which resolve by name rather than by SQL reference. +pub(crate) fn is_named_kind(kind: DeclarationKind) -> bool { + !matches!( + kind, + DeclarationKind::Macro | DeclarationKind::Enum | DeclarationKind::Constant + ) +} + +/// How placement messages name a scope. +pub(crate) fn scope_label(scope: ScopeKind) -> &'static str { + match scope { + ScopeKind::Global => "project", + ScopeKind::Inherited => "descendant-public", + ScopeKind::Local => "exact-owner-private", + ScopeKind::Private => "model-private", + } +} diff --git a/crates/sqlbuild-scopes/src/placement/constants.rs b/crates/sqlbuild-scopes/src/placement/constants.rs new file mode 100644 index 0000000000..95f31a0a02 --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/constants.rs @@ -0,0 +1,14 @@ +//! Folder names and labels the placement rules read and print. + +/// The project root as a placement anchor path. +pub(crate) const PROJECT_ROOT_PATH: &str = "."; +/// The folder grouping declaration roles below a non-root owner. +pub(crate) const DECLARATION_GROUP_DIRECTORY: &str = "_sqlbuild"; +/// Owners whose declaration roles sit directly below them. +pub(crate) const AUTHORED_ROOT_PATHS: &[&str] = &[ + "models", + "tests/unit", + "tests/scenarios", + "functions/sql", + "sources", +]; diff --git a/crates/sqlbuild-scopes/src/placement/main/declaration_directory.rs b/crates/sqlbuild-scopes/src/placement/main/declaration_directory.rs new file mode 100644 index 0000000000..cc8d87f169 --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/main/declaration_directory.rs @@ -0,0 +1,27 @@ +//! The project-relative folder where a declaration with a given placement lives. + +use crate::placement::constants::{AUTHORED_ROOT_PATHS, DECLARATION_GROUP_DIRECTORY}; +use crate::scope_index::models::{DeclarationKind, ScopeKind}; + +/// Python's `declaration_directory`. +#[must_use] +pub fn declaration_directory( + kind: DeclarationKind, + scope: ScopeKind, + path: Option<&str>, +) -> String { + let parts: &[&str] = kind.role_parts(); + let Some(path) = path.filter(|_| scope != ScopeKind::Global) else { + return parts.join("/"); + }; + let prefix: &str = if scope == ScopeKind::Local { "_" } else { "" }; + let mut role: String = format!("{prefix}{}", parts[0]); + for part in &parts[1..] { + role.push('/'); + role.push_str(part); + } + if AUTHORED_ROOT_PATHS.contains(&path) { + return format!("{path}/{role}"); + } + format!("{path}/{DECLARATION_GROUP_DIRECTORY}/{role}") +} diff --git a/crates/sqlbuild-scopes/src/placement/main/mod.rs b/crates/sqlbuild-scopes/src/placement/main/mod.rs new file mode 100644 index 0000000000..7b79db2f39 --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/main/mod.rs @@ -0,0 +1,4 @@ +pub mod declaration_directory; +pub mod placement_diagnostics; +pub mod relocate_for_resource_move; +pub mod required_declaration_placements; diff --git a/crates/sqlbuild-scopes/src/placement/main/placement_diagnostics.rs b/crates/sqlbuild-scopes/src/placement/main/placement_diagnostics.rs new file mode 100644 index 0000000000..69cb00cfbf --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/main/placement_diagnostics.rs @@ -0,0 +1,110 @@ +//! Unused-declaration and exact-placement diagnostics of a usage-complete scope index. + +use crate::placement::_helpers::anchors::{ + AnchorSets, UsagesByDeclaration, anchor_sets, usages_by_declaration, +}; +use crate::placement::_helpers::required::{owner_uses_directly, required_placement}; +use crate::placement::_helpers::roles::scope_label; +use crate::placement::main::declaration_directory::declaration_directory; +use crate::placement::models::{ + PlacementCode, PlacementDiagnostic, PlacementIndex, PlacementUsage, RequiredPlacement, +}; +use crate::scope_index::main::format_identity::format_identity; +use crate::scope_index::models::{ + DeclarationKind, DeclarationRecord, QualifiedIdentity, ScopeKind, +}; + +/// Each declaration's diagnostic, in declaration order, as `build_placement_validated_index`. +#[must_use] +pub fn placement_diagnostics(index: &PlacementIndex) -> Vec { + let usages: UsagesByDeclaration<'_> = usages_by_declaration(index); + let anchors: AnchorSets = anchor_sets(index, &usages); + let mut diagnostics: Vec = Vec::new(); + for (position, record) in index.declarations.iter().enumerate() { + let declaration_usages: &[&PlacementUsage] = usages + .get(&record.identity) + .map(Vec::as_slice) + .unwrap_or_default(); + let used: bool = if record.scope == ScopeKind::Private { + declaration_usages + .iter() + .any(|usage| owner_uses_directly(record, usage)) + } else { + !declaration_usages.is_empty() + }; + if !used { + if record.identity.kind != DeclarationKind::SingularAudit { + diagnostics.push(PlacementDiagnostic { + declaration: position, + code: PlacementCode::UnusedDeclaration, + message: format!( + "Unused {} declaration '{}' at {}; remove it or add a genuine runtime use", + record.scope.as_str(), + format_identity(&QualifiedIdentity::Declaration(record.identity.clone())), + record.path + ), + }); + } + continue; + } + if record.scope == ScopeKind::Private { + continue; + } + let Some(required) = required_placement(record, &usages, &anchors) else { + continue; + }; + if record.scope == required.scope && record.owning_path == required.path { + continue; + } + let code: PlacementCode = match (required.scope, record.scope) { + (ScopeKind::Global, _) => PlacementCode::RequiresGlobalPlacement, + (_, ScopeKind::Local) => PlacementCode::LocalNeededByDescendant, + (_, ScopeKind::Global) => PlacementCode::OverBroadGlobal, + _ => PlacementCode::OverBroadInherited, + }; + diagnostics.push(PlacementDiagnostic { + declaration: position, + code, + message: placement_message(record, &required), + }); + } + diagnostics +} + +/// `_message`: where the declaration is, where its consumers require it, and where to move it. +fn placement_message(record: &DeclarationRecord, required: &RequiredPlacement) -> String { + let current_path: &str = record + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(&record.ownership_root.path); + let directory: String = declaration_directory( + record.identity.kind, + required.scope, + required.path.as_deref(), + ); + let target: String = if required.scope == ScopeKind::Global { + format!("top-level {directory}/") + } else { + format!("{directory}/") + }; + let consumer_label: &str = if record.identity.kind == DeclarationKind::SingularAudit { + "References" + } else { + "Consumers" + }; + let required_path: &str = required + .path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or("top-level root"); + format!( + "Declaration '{}' is currently {} at '{current_path}' ({}); required {} at \ + '{required_path}'. {consumer_label}: {}. Move it to '{target}'", + format_identity(&QualifiedIdentity::Declaration(record.identity.clone())), + scope_label(record.scope), + record.path, + scope_label(required.scope), + required.consumers.join(", ") + ) +} diff --git a/crates/sqlbuild-scopes/src/placement/main/relocate_for_resource_move.rs b/crates/sqlbuild-scopes/src/placement/main/relocate_for_resource_move.rs new file mode 100644 index 0000000000..2ff103097f --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/main/relocate_for_resource_move.rs @@ -0,0 +1,121 @@ +//! The declarations whose placement follows a resource to a new path. + +use std::collections::{HashMap, HashSet}; + +use crate::placement::_helpers::anchors::{ + AnchorSets, UsagesByDeclaration, anchor_sets, usages_by_declaration, +}; +use crate::placement::_helpers::required::required_placement; +use crate::placement::main::declaration_directory::declaration_directory; +use crate::placement::main::placement_diagnostics::placement_diagnostics; +use crate::placement::models::{PlacementIndex, Relocation, RequiredPlacement}; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, OwnershipRoot, OwnershipRootKind, ResourceIdentity, + ScopeKind, +}; +use sqlbuild_core::posix_path::main::posix_name::posix_name; + +/// Python's `relocate_for_resource_move`: each moved declaration, or `None` when unsettled. +#[must_use] +pub fn relocate_for_resource_move( + index: &PlacementIndex, + resource: &ResourceIdentity, + destination: &str, +) -> Option> { + let misplaced: HashSet<&DeclarationIdentity> = placement_diagnostics(index) + .into_iter() + .map(|diagnostic| &index.declarations[diagnostic.declaration].identity) + .collect(); + let mut current: PlacementIndex = index.clone(); + for record in &mut current.resources { + if &record.identity == resource { + record.path = destination.to_owned(); + } + } + for _ in 0..=index.declarations.len() { + let usages: UsagesByDeclaration<'_> = usages_by_declaration(¤t); + let anchors: AnchorSets = anchor_sets(¤t, &usages); + let mut relocated: HashMap = HashMap::new(); + for record in ¤t.declarations { + if record.scope == ScopeKind::Private || misplaced.contains(&record.identity) { + continue; + } + if let Some(moved) = relocated_record(record, &usages, &anchors) { + relocated.insert(record.identity.clone(), moved); + } + } + if relocated.is_empty() { + return Some(changed_records(index, ¤t)); + } + for record in &mut current.declarations { + if let Some(moved) = relocated.get(&record.identity) { + *record = moved.clone(); + } + } + } + None +} + +/// `_relocated_record`: the record at its required placement, when that differs. +fn relocated_record( + record: &DeclarationRecord, + usages: &UsagesByDeclaration<'_>, + anchors: &AnchorSets, +) -> Option { + let RequiredPlacement { scope, path, .. } = required_placement(record, usages, anchors)?; + if record.scope == scope && record.owning_path == path { + return None; + } + let directory: String = declaration_directory(record.identity.kind, scope, path.as_deref()); + let roots: HashSet<&OwnershipRoot> = anchors + .get(&record.identity) + .into_iter() + .flatten() + .map(|(root, _)| root) + .collect(); + let single_root: Option<&OwnershipRoot> = match roots.iter().next() { + Some(root) if roots.len() == 1 && scope != ScopeKind::Global => Some(root), + _ => None, + }; + let ownership_root: OwnershipRoot = match single_root { + Some(root) => (*root).clone(), + None => OwnershipRoot { + path: directory.clone(), + kind: OwnershipRootKind::Global, + resource_kind: None, + }, + }; + Some(DeclarationRecord { + path: format!("{directory}/{}", posix_name(&record.path)), + scope, + ownership_root, + owning_path: if scope == ScopeKind::Global { + None + } else { + path + }, + ..record.clone() + }) +} + +/// Each declaration whose path differs from the last original record of its identity. +fn changed_records(original: &PlacementIndex, current: &PlacementIndex) -> Vec { + let paths: HashMap<&DeclarationIdentity, &str> = original + .declarations + .iter() + .map(|record| (&record.identity, record.path.as_str())) + .collect(); + current + .declarations + .iter() + .enumerate() + .filter(|(_, record)| paths.get(&record.identity) != Some(&record.path.as_str())) + .map(|(position, record)| Relocation { + declaration: position, + path: record.path.clone(), + scope: record.scope, + ownership_root: record.ownership_root.clone(), + owning_path: record.owning_path.clone(), + }) + .collect() +} diff --git a/crates/sqlbuild-scopes/src/placement/main/required_declaration_placements.rs b/crates/sqlbuild-scopes/src/placement/main/required_declaration_placements.rs new file mode 100644 index 0000000000..0a61d10603 --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/main/required_declaration_placements.rs @@ -0,0 +1,30 @@ +//! The placement every declaration's consumers require, resolved over one shared anchor pass. + +use std::collections::{HashMap, HashSet}; + +use crate::placement::_helpers::anchors::{ + AnchorSets, UsagesByDeclaration, anchor_sets, usages_by_declaration, +}; +use crate::placement::_helpers::required::required_placement; +use crate::placement::models::{PlacementIndex, RequiredPlacement}; +use crate::scope_index::models::DeclarationIdentity; + +/// Python's `resolve_required_placement` for each identity's first record, where it has one. +#[must_use] +pub fn required_declaration_placements( + index: &PlacementIndex, +) -> HashMap { + let usages: UsagesByDeclaration<'_> = usages_by_declaration(index); + let anchors: AnchorSets = anchor_sets(index, &usages); + let mut placements: HashMap = HashMap::new(); + let mut seen: HashSet<&DeclarationIdentity> = HashSet::new(); + for record in &index.declarations { + if !seen.insert(&record.identity) { + continue; + } + if let Some(placement) = required_placement(record, &usages, &anchors) { + placements.insert(record.identity.clone(), placement); + } + } + placements +} diff --git a/crates/sqlbuild-scopes/src/placement/mod.rs b/crates/sqlbuild-scopes/src/placement/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-scopes/src/placement/models.rs b/crates/sqlbuild-scopes/src/placement/models.rs new file mode 100644 index 0000000000..25a1b7417c --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/models.rs @@ -0,0 +1,71 @@ +//! The scope facts placement rules read and the placements and diagnostics they derive. + +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, OwnershipRoot, QualifiedIdentity, ResourceRecord, + ScopeKind, +}; + +/// Python's `UsageRecord`, without the kind and enum member placement ignores. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct PlacementUsage { + pub consumer: QualifiedIdentity, + pub declaration: DeclarationIdentity, + pub through: Option, +} + +/// The resources, declarations and runtime usages of a usage-complete scope index. +#[derive(Clone, Debug, Default)] +pub struct PlacementIndex { + pub resources: Vec, + pub declarations: Vec, + pub usages: Vec, +} + +/// The placement diagnostic codes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PlacementCode { + LocalNeededByDescendant, + OverBroadInherited, + RequiresGlobalPlacement, + UnusedDeclaration, + OverBroadGlobal, +} + +impl PlacementCode { + /// Python's `ScopeDiagnosticCode` value. + pub fn as_str(self) -> &'static str { + match self { + Self::LocalNeededByDescendant => "S007", + Self::OverBroadInherited => "S008", + Self::RequiresGlobalPlacement => "S009", + Self::UnusedDeclaration => "S010", + Self::OverBroadGlobal => "S024", + } + } +} + +/// A misplaced or unused declaration, by its position in the index's declarations. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PlacementDiagnostic { + pub declaration: usize, + pub code: PlacementCode, + pub message: String, +} + +/// Where a declaration's consumers require it, and their sorted labels. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RequiredPlacement { + pub scope: ScopeKind, + pub path: Option, + pub consumers: Vec, +} + +/// A declaration's placement after a resource move, by its position in the index. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Relocation { + pub declaration: usize, + pub path: String, + pub scope: ScopeKind, + pub ownership_root: OwnershipRoot, + pub owning_path: Option, +} diff --git a/crates/sqlbuild-scopes/src/placement/tests.rs b/crates/sqlbuild-scopes/src/placement/tests.rs new file mode 100644 index 0000000000..b4ee4400cf --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_placement.rs"] +mod placement; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-scopes/src/placement/tests/helpers.rs b/crates/sqlbuild-scopes/src/placement/tests/helpers.rs new file mode 100644 index 0000000000..e99edb192f --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/tests/helpers.rs @@ -0,0 +1,110 @@ +use crate::placement::models::{PlacementIndex, PlacementUsage}; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationKind, DeclarationRecord, OwnershipRoot, OwnershipRootKind, + QualifiedIdentity, ResourceIdentity, ResourceKind, ResourceRecord, ScopeKind, +}; + +pub(super) fn model(name: &str) -> ResourceIdentity { + ResourceIdentity { + kind: ResourceKind::Model, + name: name.to_owned(), + } +} + +fn macro_identity(name: &str) -> DeclarationIdentity { + DeclarationIdentity { + kind: DeclarationKind::Macro, + name: name.to_owned(), + owner: None, + } +} + +fn models_root() -> OwnershipRoot { + OwnershipRoot { + path: "models".to_owned(), + kind: OwnershipRootKind::Resource, + resource_kind: Some(ResourceKind::Model), + } +} + +fn model_record(name: &str, path: &str) -> ResourceRecord { + ResourceRecord { + identity: model(name), + path: path.to_owned(), + ownership_root: models_root(), + } +} + +/// A macro declared at `path` with `scope` and `owning_path` below the models root. +fn macro_record(name: &str, path: &str, placement: (ScopeKind, Option<&str>)) -> DeclarationRecord { + let (scope, owning_path) = placement; + DeclarationRecord { + identity: macro_identity(name), + path: path.to_owned(), + line: 1, + column: 0, + scope, + ownership_root: models_root(), + owning_path: owning_path.map(str::to_owned), + dependencies: Vec::new(), + } +} + +fn runtime(consumer: &str, declaration: &str) -> PlacementUsage { + PlacementUsage { + consumer: QualifiedIdentity::Resource(model(consumer)), + declaration: macro_identity(declaration), + through: None, + } +} + +const SALES: (ScopeKind, Option<&str>) = (ScopeKind::Local, Some("models/sales")); + +/// A model and a descendant-folder model, a local macro both use, and an unused global macro. +pub(super) fn descendant_models() -> PlacementIndex { + PlacementIndex { + resources: vec![ + model_record("orders", "models/sales/orders.sql"), + model_record("customers", "models/sales/crm/customers.sql"), + ], + declarations: vec![ + macro_record("cents", "models/sales/_macros/cents.py", SALES), + macro_record("unused", "macros/unused.py", (ScopeKind::Global, None)), + macro_record("shared", "models/sales/_macros/shared.py", SALES), + ], + usages: vec![ + runtime("orders", "cents"), + runtime("orders", "shared"), + runtime("customers", "shared"), + ], + } +} + +/// One model using a local macro, which a further macro uses in turn. +pub(super) fn chained_macros() -> PlacementIndex { + PlacementIndex { + resources: vec![model_record("orders", "models/sales/orders.sql")], + declarations: vec![ + macro_record("cents", "models/sales/_macros/cents.py", SALES), + macro_record("round", "models/sales/_macros/round.py", SALES), + ], + usages: vec![ + runtime("orders", "cents"), + PlacementUsage { + consumer: QualifiedIdentity::Declaration(macro_identity("cents")), + declaration: macro_identity("round"), + through: None, + }, + ], + } +} + +/// Owned `(declaration, path, owning path)` rows of expected relocations. +pub(super) fn owned_relocations( + items: &[(usize, &str, Option<&str>)], +) -> Vec<(usize, String, Option)> { + items + .iter() + .map(|(position, path, owning)| (*position, (*path).to_owned(), owning.map(str::to_owned))) + .collect() +} diff --git a/crates/sqlbuild-scopes/src/placement/tests/test_placement.rs b/crates/sqlbuild-scopes/src/placement/tests/test_placement.rs new file mode 100644 index 0000000000..cfecc3a23f --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/tests/test_placement.rs @@ -0,0 +1,149 @@ +use crate::placement::main::declaration_directory::declaration_directory; +use crate::placement::main::placement_diagnostics::placement_diagnostics; +use crate::placement::main::relocate_for_resource_move::relocate_for_resource_move; +use crate::placement::models::Relocation; +use crate::placement::tests::helpers::{ + chained_macros, descendant_models, model, owned_relocations, +}; +use crate::placement::tests::test_types::{ + DeclarationDirectoryTestCase, PlacementDiagnosticsTestCase, RelocationTestCase, +}; +use crate::scope_index::models::{DeclarationKind, ScopeKind}; + +#[test] +fn given_usages_when_validating_placement_then_unused_and_misplaced_declarations_are_reported() { + let test_cases = [ + PlacementDiagnosticsTestCase { + description: "an unused global and a local macro needed by a descendant folder", + index: descendant_models, + expected_diagnostics: &[ + ( + 1, + "S010", + "Unused global declaration 'macro:unused' at macros/unused.py; remove it or \ + add a genuine runtime use", + ), + ( + 2, + "S007", + "Declaration 'macro:shared' is currently exact-owner-private at \ + 'models/sales' (models/sales/_macros/shared.py); required descendant-public \ + at 'models/sales'. Consumers: model:customers, model:orders. Move it to \ + 'models/sales/_sqlbuild/macros/'", + ), + ], + }, + PlacementDiagnosticsTestCase { + description: "a macro used only through another macro inherits its anchors", + index: chained_macros, + expected_diagnostics: &[], + }, + ]; + for test_case in test_cases { + let diagnostics: Vec<(usize, &str, String)> = placement_diagnostics(&(test_case.index)()) + .into_iter() + .map(|item| (item.declaration, item.code.as_str(), item.message)) + .collect(); + + assert_eq!( + diagnostics, + test_case + .expected_diagnostics + .iter() + .map(|(position, code, message)| (*position, *code, (*message).to_owned())) + .collect::>(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_placement_when_choosing_directory_then_role_folders_match_python() { + let test_cases = [ + DeclarationDirectoryTestCase { + description: "a global macro", + kind: DeclarationKind::Macro, + scope: ScopeKind::Global, + path: Some("models/sales"), + expected_directory: "macros", + }, + DeclarationDirectoryTestCase { + description: "a local audit below an authored root", + kind: DeclarationKind::Audit, + scope: ScopeKind::Local, + path: Some("models"), + expected_directory: "models/_audits/generic", + }, + DeclarationDirectoryTestCase { + description: "an inherited hook below a nested folder", + kind: DeclarationKind::SqlHook, + scope: ScopeKind::Inherited, + path: Some("models/sales"), + expected_directory: "models/sales/_sqlbuild/hooks/sql", + }, + DeclarationDirectoryTestCase { + description: "an inherited enum without a folder", + kind: DeclarationKind::Enum, + scope: ScopeKind::Inherited, + path: None, + expected_directory: "enums", + }, + ]; + for test_case in test_cases { + assert_eq!( + declaration_directory(test_case.kind, test_case.scope, test_case.path), + test_case.expected_directory, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_resource_move_when_relocating_then_chained_local_declarations_follow() { + let test_cases = [ + RelocationTestCase { + description: "a model and the macro chain it uses move to a new folder", + index: chained_macros, + destination: "models/finance/orders.sql", + expected_relocations: Some(&[ + ( + 0, + "models/finance/_sqlbuild/_macros/cents.py", + Some("models/finance"), + ), + ( + 1, + "models/finance/_sqlbuild/_macros/round.py", + Some("models/finance"), + ), + ]), + }, + RelocationTestCase { + description: "a move within the same folder relocates nothing", + index: chained_macros, + destination: "models/sales/renamed.sql", + expected_relocations: Some(&[]), + }, + ]; + for test_case in test_cases { + let relocations: Option> = relocate_for_resource_move( + &(test_case.index)(), + &model("orders"), + test_case.destination, + ); + + assert_eq!( + relocations.map(|items| { + items + .into_iter() + .map(|item| (item.declaration, item.path, item.owning_path)) + .collect::>() + }), + test_case.expected_relocations.map(owned_relocations), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/placement/tests/test_types.rs b/crates/sqlbuild-scopes/src/placement/tests/test_types.rs new file mode 100644 index 0000000000..070617d7c9 --- /dev/null +++ b/crates/sqlbuild-scopes/src/placement/tests/test_types.rs @@ -0,0 +1,29 @@ +use crate::placement::models::PlacementIndex; +use crate::scope_index::models::{DeclarationKind, ScopeKind}; + +/// A usage-complete index and the `(declaration, code, message)` diagnostics it raises. +pub(super) struct PlacementDiagnosticsTestCase { + pub(super) description: &'static str, + pub(super) index: fn() -> PlacementIndex, + pub(super) expected_diagnostics: &'static [(usize, &'static str, &'static str)], +} + +/// A placement and the folder a declaration with it lives in. +pub(super) struct DeclarationDirectoryTestCase { + pub(super) description: &'static str, + pub(super) kind: DeclarationKind, + pub(super) scope: ScopeKind, + pub(super) path: Option<&'static str>, + pub(super) expected_directory: &'static str, +} + +/// `(declaration, new path, owning path)` of one relocated declaration. +pub(super) type RelocationCase = (usize, &'static str, Option<&'static str>); + +/// A resource move and each declaration it relocates. +pub(super) struct RelocationTestCase { + pub(super) description: &'static str, + pub(super) index: fn() -> PlacementIndex, + pub(super) destination: &'static str, + pub(super) expected_relocations: Option<&'static [RelocationCase]>, +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/_helpers/facts.rs b/crates/sqlbuild-scopes/src/relationship_names/_helpers/facts.rs new file mode 100644 index 0000000000..23fbc065cb --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/_helpers/facts.rs @@ -0,0 +1,80 @@ +//! Each test block's expected models, called macros and tested macros. + +use sqlbuild_core::text::models::PythonText; +use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::relationship_names::_helpers::cte_scan::{ScanText, expected_names}; +use crate::relationship_names::constants::{ + DECLARATION_CALLS, MACRO_ACTUAL_CTE, MACRO_MODE, MODEL_MODE, +}; +use crate::relationship_names::main::top_level_ctes::scan_top_level_ctes; +use crate::relationship_names::models::{ + RelationshipSource, TestBlockInput, TestFileInput, TopLevelCtes, +}; +use crate::scope_index::models::{RelationshipFact, ResourceIdentity, ResourceKind}; + +/// Every block's expected models: model blocks scanned, other modes none. +pub(crate) fn scanned_names( + file: &TestFileInput<'_>, + syntax: &LexicalSyntax, +) -> Vec, String>> { + file.blocks + .iter() + .map(|block| { + if block.mode != MODEL_MODE { + return Ok(Vec::new()); + } + let text: ScanText<'_> = ScanText { + sql: block.sql, + file: file.relative_path, + source: RelationshipSource::Test, + }; + expected_names(&text, syntax) + }) + .collect() +} + +/// One block's fact: expected models, then nested macro calls, then a macro test's tested macros. +pub(crate) fn block_fact( + block: &TestBlockInput<'_>, + names: Result, String>, + relative_path: &str, + rules: (&LexicalSyntax, PythonText), +) -> Result { + let (syntax, python) = rules; + let expected_models: Vec = names?; + let called_macros: Vec = call_names(python, block.sql, true)?; + let tested_macros: Vec = if block.mode == MACRO_MODE { + let ctes: Vec<(String, String)> = + match scan_top_level_ctes(block.sql, relative_path, RelationshipSource::Test, syntax) { + TopLevelCtes::Scanned(ctes) => ctes, + TopLevelCtes::Failed(message) => return Err(message), + }; + match ctes.iter().find(|(name, _)| name == MACRO_ACTUAL_CTE) { + Some((_, body)) => call_names(python, body, false)?, + None => Vec::new(), + } + } else { + Vec::new() + }; + Ok(RelationshipFact { + resource: ResourceIdentity { + kind: ResourceKind::Test, + name: block.name.to_owned(), + }, + expected_models, + called_macros, + tested_macros, + }) +} + +/// `find_macro_call_names` or `find_nested_macro_call_names`, or the scan's message. +fn call_names(python: PythonText, sql: &str, nested: bool) -> Result, String> { + let names: Vec = + macro_call_names(python, sql, nested).map_err(|error| error.message().to_owned())?; + Ok(names + .into_iter() + .filter(|name| !DECLARATION_CALLS.contains(&name.as_str())) + .collect()) +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/_helpers/mod.rs b/crates/sqlbuild-scopes/src/relationship_names/_helpers/mod.rs index 7671fab062..458fb4a4a3 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/_helpers/mod.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod cte_scan; +pub(crate) mod facts; diff --git a/crates/sqlbuild-scopes/src/relationship_names/constants.rs b/crates/sqlbuild-scopes/src/relationship_names/constants.rs new file mode 100644 index 0000000000..6c272698ff --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/constants.rs @@ -0,0 +1,7 @@ +//! Fixed names the relationship scan reads. + +pub(crate) const MODEL_MODE: &str = "model"; +pub(crate) const MACRO_MODE: &str = "macro"; +pub(crate) const MACRO_ACTUAL_CTE: &str = "__macro_actual__"; +/// Declaration references, which macro call scans skip. +pub(crate) const DECLARATION_CALLS: [&str; 2] = ["enum", "const"]; diff --git a/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs b/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs index 948c16b2bf..ab3952ec5d 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/main/mod.rs @@ -1,2 +1,4 @@ pub mod expected_model_names; +pub mod relationship_facts; +pub mod stored_expected_names; pub mod top_level_ctes; diff --git a/crates/sqlbuild-scopes/src/relationship_names/main/relationship_facts.rs b/crates/sqlbuild-scopes/src/relationship_names/main/relationship_facts.rs new file mode 100644 index 0000000000..35d128ba34 --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/main/relationship_facts.rs @@ -0,0 +1,60 @@ +//! The relationship facts of every SQL test block and scenario, and the faults of the rest. + +use sqlbuild_core::text::models::PythonText; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::relationship_names::_helpers::cte_scan::{ScanText, expected_names}; +use crate::relationship_names::_helpers::facts::{block_fact, scanned_names}; +use crate::relationship_names::models::{ + RelationshipScan, RelationshipSource, ScenarioInput, TestFileInput, +}; +use crate::scope_index::models::{RelationshipFact, ResourceIdentity, ResourceKind}; + +/// Each test block's and scenario's expected models, called and tested macros, or its fault. +#[must_use] +pub fn relationship_facts( + files: &[TestFileInput<'_>], + scenarios: &[ScenarioInput<'_>], + syntax: &LexicalSyntax, + python: PythonText, +) -> RelationshipScan { + let mut scan: RelationshipScan = RelationshipScan::default(); + for (position, file) in files.iter().enumerate() { + let names: Vec, String>> = match &file.stored { + Some(stored) => stored.iter().cloned().map(Ok).collect(), + None => scanned_names(file, syntax), + }; + if file.stored.is_none() && names.iter().all(Result::is_ok) { + let complete: Vec> = names.iter().flatten().cloned().collect(); + scan.scanned_files.push((position, complete)); + } + for (block, names) in file.blocks.iter().zip(names) { + match block_fact(block, names, file.relative_path, (syntax, python)) { + Ok(fact) => scan.facts.push(fact), + Err(message) => scan.faults.push((file.relative_path.to_owned(), message)), + } + } + } + for scenario in scenarios { + let text: ScanText<'_> = ScanText { + sql: scenario.sql, + file: scenario.relative_path, + source: RelationshipSource::Scenario, + }; + match expected_names(&text, syntax) { + Ok(expected_models) => scan.facts.push(RelationshipFact { + resource: ResourceIdentity { + kind: ResourceKind::Scenario, + name: scenario.name.to_owned(), + }, + expected_models, + called_macros: Vec::new(), + tested_macros: Vec::new(), + }), + Err(message) => scan + .faults + .push((scenario.relative_path.to_owned(), message)), + } + } + scan +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/main/stored_expected_names.rs b/crates/sqlbuild-scopes/src/relationship_names/main/stored_expected_names.rs new file mode 100644 index 0000000000..33384b17ca --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/main/stored_expected_names.rs @@ -0,0 +1,11 @@ +//! One test file's stored expected-model names, read back from the shared scan store. + +/// Every block's names when `value` is exactly a list of `block_count` lists of strings. +#[must_use] +pub fn stored_expected_names(value: &[u8], block_count: usize) -> Option>> { + let stored: Vec> = match serde_json::from_slice(value) { + Ok(stored) => stored, + Err(_) => return None, + }; + (stored.len() == block_count).then_some(stored) +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/mod.rs b/crates/sqlbuild-scopes/src/relationship_names/mod.rs index e6e5a52eb4..1e64f3d116 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/mod.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-scopes/src/relationship_names/models.rs b/crates/sqlbuild-scopes/src/relationship_names/models.rs index cb236142d9..88c64a225e 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/models.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/models.rs @@ -26,3 +26,41 @@ pub enum TopLevelCtes { /// The message the scanner raises. Failed(String), } + +/// One SQL test block a relationship fact is read from. +#[derive(Clone, Copy, Debug)] +pub struct TestBlockInput<'a> { + /// The block's name, or its file's stem. + pub name: &'a str, + pub sql: &'a str, + /// The block's `SqlTestMode` value. + pub mode: &'a str, +} + +/// One SQL test file, with the expected-model names a previous compile stored for it. +#[derive(Clone, Debug)] +pub struct TestFileInput<'a> { + pub relative_path: &'a str, + pub blocks: Vec>, + /// Every block's names, reused instead of scanning the file again. + pub stored: Option>>, +} + +/// One SQL scenario a relationship fact is read from. +#[derive(Clone, Copy, Debug)] +pub struct ScenarioInput<'a> { + pub name: &'a str, + pub relative_path: &'a str, + pub sql: &'a str, +} + +/// The facts of every test block and scenario that scanned, and the faults of the rest. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct RelationshipScan { + /// Facts of tests, then scenarios, in authored order. + pub facts: Vec, + /// `(relative path, message)` of each block or scenario that failed, tests first. + pub faults: Vec<(String, String)>, + /// Each unstored test file, by position, with every block's names when none failed. + pub scanned_files: Vec<(usize, Vec>)>, +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests.rs b/crates/sqlbuild-scopes/src/relationship_names/tests.rs index 12cfec4f9b..716010ccf0 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/tests.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/tests.rs @@ -2,5 +2,7 @@ mod expected_model_names; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_relationship_facts.rs"] +mod relationship_facts; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs b/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs index e71769b8f2..2b8b1eb9a0 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/tests/helpers.rs @@ -73,3 +73,22 @@ pub(super) fn scanned_ctes(ctes: &[(&str, &str)]) -> TopLevelCtes { fn owned(values: &[&str]) -> Vec { values.iter().map(|value| (*value).to_owned()).collect() } + +/// `kind:name expected=[..] called=[..] tested=[..]`. +pub(super) fn fact_summary(fact: &crate::scope_index::models::RelationshipFact) -> String { + format!( + "{}:{} expected=[{}] called=[{}] tested=[{}]", + fact.resource.kind.as_str(), + fact.resource.name, + fact.expected_models.join(", "), + fact.called_macros.join(", "), + fact.tested_macros.join(", ") + ) +} + +pub(super) fn owned_names(names: &[&[&str]]) -> Vec> { + names + .iter() + .map(|block| block.iter().map(|name| (*name).to_owned()).collect()) + .collect() +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests/test_relationship_facts.rs b/crates/sqlbuild-scopes/src/relationship_names/tests/test_relationship_facts.rs new file mode 100644 index 0000000000..454190403e --- /dev/null +++ b/crates/sqlbuild-scopes/src/relationship_names/tests/test_relationship_facts.rs @@ -0,0 +1,268 @@ +use crate::relationship_names::main::expected_model_names::expected_model_names; +use crate::relationship_names::main::relationship_facts::relationship_facts; +use crate::relationship_names::main::stored_expected_names::stored_expected_names; +use crate::relationship_names::models::{ + ExpectedNames, RelationshipScan, RelationshipSource, ScenarioInput, TestBlockInput, + TestFileInput, +}; +use crate::relationship_names::tests::helpers::{fact_summary, generic_syntax, owned_names}; +use crate::relationship_names::tests::test_types::{ + RelationshipFactsTestCase, ScanMessageTestCase, StoredNamesTestCase, +}; +use sqlbuild_core::text::main::active_python_text::active_python_text; + +#[test] +fn given_tests_and_scenarios_when_reading_relationship_facts_then_python_order_and_faults_hold() { + let test_cases = [ + RelationshipFactsTestCase { + description: "model, macro and udf blocks with nested calls", + files: &[( + "tests/unit/orders.sql", + &[ + ( + "orders_test", + "WITH __ref__a AS (SELECT @outer(@inner(1))), __expected__orders AS (SELECT @enum('s').A) SELECT 1", + "model", + ), + ( + "macro_test", + "WITH __macro_actual__ AS (SELECT @cents(1), @const('c')), __macro_expected__ AS (SELECT 100)", + "macro", + ), + ( + "udf_test", + "WITH __udf_actual__ AS (SELECT 1), __udf_expected__ AS (SELECT 1)", + "udf", + ), + ], + None, + )], + scenarios: &[( + "orders__paid", + "tests/scenarios/orders__paid.sql", + "WITH __expected__orders AS (SELECT 1), __expected__customers AS (SELECT 2)", + )], + expected_facts: &[ + "test:orders_test expected=[orders] called=[outer, inner] tested=[]", + "test:macro_test expected=[] called=[cents] tested=[cents]", + "test:udf_test expected=[] called=[] tested=[]", + "scenario:orders__paid expected=[orders, customers] called=[] tested=[]", + ], + expected_faults: &[], + expected_stored: &[0], + }, + RelationshipFactsTestCase { + description: "stored names are reused and the file is not stored again", + files: &[( + "tests/unit/orders.sql", + &[( + "orders_test", + "WITH __expected__orders AS (SELECT 1) SELECT 1", + "model", + )], + Some(&[&["stored_model"]]), + )], + scenarios: &[], + expected_facts: &["test:orders_test expected=[stored_model] called=[] tested=[]"], + expected_faults: &[], + expected_stored: &[], + }, + RelationshipFactsTestCase { + description: "a failing block keeps its file unstored and the other blocks' facts", + files: &[( + "tests/unit/orders.sql", + &[ + ("broken", "WITH __expected__orders AS (SELECT 1", "model"), + ( + "fine", + "WITH __expected__customers AS (SELECT 1) SELECT 1", + "model", + ), + ( + "bad_call", + "WITH __macro_actual__ AS (SELECT @m(1, $$x) ), __macro_expected__ AS (SELECT 1)", + "macro", + ), + ], + None, + )], + scenarios: &[( + "broken_scenario", + "tests/scenarios/broken.sql", + "WITH __expected__a AS (SELECT 1", + )], + expected_facts: &["test:fine expected=[customers] called=[] tested=[]"], + expected_faults: &[ + ( + "tests/unit/orders.sql", + "SQL test contains an unclosed parenthesis", + ), + ( + "tests/unit/orders.sql", + "Macro expansion contains an unclosed quoted string", + ), + ( + "tests/scenarios/broken.sql", + "SQL scenario contains an unclosed parenthesis", + ), + ], + expected_stored: &[], + }, + ]; + let syntax = generic_syntax(); + for test_case in test_cases { + let files: Vec> = test_case + .files + .iter() + .map(|(relative_path, blocks, stored)| TestFileInput { + relative_path, + blocks: blocks + .iter() + .map(|(name, sql, mode)| TestBlockInput { name, sql, mode }) + .collect(), + stored: stored.map(owned_names), + }) + .collect(); + let scenarios: Vec> = test_case + .scenarios + .iter() + .map(|(name, relative_path, sql)| ScenarioInput { + name, + relative_path, + sql, + }) + .collect(); + + let scan: RelationshipScan = + relationship_facts(&files, &scenarios, &syntax, active_python_text()); + + assert_eq!( + ( + scan.facts.iter().map(fact_summary).collect::>(), + scan.faults + .iter() + .map(|(path, message)| (path.as_str(), message.as_str())) + .collect::>(), + scan.scanned_files + .iter() + .map(|(position, _)| *position) + .collect::>(), + ), + ( + test_case + .expected_facts + .iter() + .map(|fact| (*fact).to_owned()) + .collect::>(), + test_case.expected_faults.to_vec(), + test_case.expected_stored.to_vec(), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_stored_bytes_when_decoding_expected_names_then_only_exact_shapes_are_reused() { + let test_cases = [ + StoredNamesTestCase { + description: "one list per block", + value: r#"[["orders"],[]]"#, + block_count: 2, + expected_names: Some(&[&["orders"], &[]]), + }, + StoredNamesTestCase { + description: "a different block count", + value: r#"[["orders"]]"#, + block_count: 2, + expected_names: None, + }, + StoredNamesTestCase { + description: "undecodable bytes", + value: "{]", + block_count: 1, + expected_names: None, + }, + StoredNamesTestCase { + description: "a misshapen entry", + value: r#"[{"mode": "model"}]"#, + block_count: 1, + expected_names: None, + }, + StoredNamesTestCase { + description: "a non-string name", + value: "[[1]]", + block_count: 1, + expected_names: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + stored_expected_names(test_case.value.as_bytes(), test_case.block_count), + test_case.expected_names.map(owned_names), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_malformed_ctes_when_scanning_expected_models_then_messages_name_the_source() { + let test_cases = [ + ScanMessageTestCase { + description: "expected model scan without AS keeps sql test wording", + source: RelationshipSource::Test, + file_label: "tests/unit/orders.sql", + sql: "WITH __source__raw_orders (SELECT 1 AS order_id) SELECT 1", + expected_message: "SQL test 'tests/unit/orders.sql' expected keyword AS", + }, + ScanMessageTestCase { + description: "expected model scan without a target keeps sql test wording", + source: RelationshipSource::Test, + file_label: "tests/unit/orders.sql", + sql: "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__ AS (SELECT 1 AS order_id) SELECT 1", + expected_message: "SQL test 'tests/unit/orders.sql' must use __expected__ to identify a target", + }, + ScanMessageTestCase { + description: "expected model scan without AS uses scenario wording", + source: RelationshipSource::Scenario, + file_label: "tests/scenarios/orders.sql", + sql: "WITH __source__raw_orders (SELECT 1 AS order_id) SELECT 1", + expected_message: "SQL scenario 'tests/scenarios/orders.sql' expected keyword AS", + }, + ScanMessageTestCase { + description: "expected model scan unclosed comment uses scenario wording", + source: RelationshipSource::Scenario, + file_label: "tests/scenarios/orders.sql", + sql: "WITH /* unfinished", + expected_message: "SQL scenario contains an unclosed block comment", + }, + ScanMessageTestCase { + description: "expected model scan leading unclosed comment uses scenario wording", + source: RelationshipSource::Scenario, + file_label: "tests/scenarios/orders.sql", + sql: "/* unfinished", + expected_message: "SQL scenario contains an unclosed block comment", + }, + ScanMessageTestCase { + description: "expected model scan without a target uses scenario wording", + source: RelationshipSource::Scenario, + file_label: "tests/scenarios/orders.sql", + sql: "WITH __source__raw_orders AS (SELECT 1 AS order_id), __expected__ AS (SELECT 1 AS order_id) SELECT 1", + expected_message: "SQL scenario 'tests/scenarios/orders.sql' must use __expected__ to identify a target", + }, + ]; + let syntax = generic_syntax(); + for test_case in test_cases { + let texts: Vec<(String, String)> = + vec![(test_case.sql.to_owned(), test_case.file_label.to_owned())]; + + assert_eq!( + expected_model_names(&texts, test_case.source, &syntax), + vec![ExpectedNames::Failed(test_case.expected_message.to_owned())], + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs b/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs index cc7681257a..fc98af31f9 100644 --- a/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs +++ b/crates/sqlbuild-scopes/src/relationship_names/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::relationship_names::models::{ExpectedNames, TopLevelCtes}; +use crate::relationship_names::models::{ExpectedNames, RelationshipSource, TopLevelCtes}; /// One SQL test body and the native expected-model outcome. pub(super) struct ExpectedNamesTestCase { @@ -30,3 +30,41 @@ pub(super) struct SyntaxTestCase { pub(super) sql: &'static str, pub(super) expected_outcome: ExpectedNames, } + +/// A test file's relative path, `(name, sql, mode)` blocks and stored names. +pub(super) type FileCase = ( + &'static str, + &'static [(&'static str, &'static str, &'static str)], + Option<&'static [&'static [&'static str]]>, +); + +/// Test files and scenarios, and the facts and faults their relationship scan reads. +pub(super) struct RelationshipFactsTestCase { + pub(super) description: &'static str, + /// `(relative path, [(name, sql, mode)], stored names)` per test file. + pub(super) files: &'static [FileCase], + /// `(name, relative path, sql)` per scenario. + pub(super) scenarios: &'static [(&'static str, &'static str, &'static str)], + /// `kind:name expected=.. called=.. tested=..` per fact. + pub(super) expected_facts: &'static [&'static str], + pub(super) expected_faults: &'static [(&'static str, &'static str)], + /// Positions of the files whose scanned names would be stored. + pub(super) expected_stored: &'static [usize], +} + +/// Stored scan bytes and whether they decode as a file of `block_count` blocks. +pub(super) struct StoredNamesTestCase { + pub(super) description: &'static str, + pub(super) value: &'static str, + pub(super) block_count: usize, + pub(super) expected_names: Option<&'static [&'static [&'static str]]>, +} + +/// A malformed test or scenario and the message its expected-model scan raises. +pub(super) struct ScanMessageTestCase { + pub(super) description: &'static str, + pub(super) source: RelationshipSource, + pub(super) file_label: &'static str, + pub(super) sql: &'static str, + pub(super) expected_message: &'static str, +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/_helpers/decode.rs b/crates/sqlbuild-scopes/src/scope_cache/_helpers/decode.rs new file mode 100644 index 0000000000..adc756f0f9 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/_helpers/decode.rs @@ -0,0 +1,374 @@ +//! Python's `decode_scope_index` validation: any fault is a cache miss. + +use serde_json::{Map, Value}; + +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::scope_cache::_helpers::json::{canonical_json, sha256_hex}; +use crate::scope_cache::constants::{ + CACHE_SCHEMA_VERSION, COMPLETENESS_KEYS, DECLARATION_IDENTITY, RESOURCE_IDENTITY, +}; +use crate::scope_cache::models::{ + CacheMiss, CachedDeclaration, CachedDiagnostic, CachedInaccessible, CachedIndex, CachedUsage, + CachedVisibility, ConstantFacts, EnumFacts, MacroFacts, +}; +use crate::scope_index::main::normalize_scope_path::normalize_scope_path; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationKind, DeclarationRecord, GrantKind, OwnershipRoot, + OwnershipRootKind, QualifiedIdentity, ReportGrant, ResourceIdentity, ResourceKind, + ResourceRecord, ScopeKind, +}; + +/// A payload Python's decoder rejects. +pub(crate) struct Miss; + +type Decoded = Result; + +fn object(value: &Value) -> Decoded<&Map> { + value.as_object().ok_or(Miss) +} + +fn list<'a>(value: &'a Map, key: &str) -> Decoded<&'a Vec> { + value.get(key).and_then(Value::as_array).ok_or(Miss) +} + +fn string(value: &Map, key: &str) -> Decoded { + value + .get(key) + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or(Miss) +} + +fn integer(value: Option<&Value>) -> Decoded { + value.and_then(Value::as_i64).ok_or(Miss) +} + +fn boolean(value: &Map, key: &str) -> Decoded { + value.get(key).and_then(Value::as_bool).ok_or(Miss) +} + +fn optional_string(value: Option<&Value>) -> Decoded> { + match value { + None | Some(Value::Null) => Ok(None), + Some(Value::String(text)) => Ok(Some(text.clone())), + Some(_) => Err(Miss), + } +} + +fn optional_integer(value: Option<&Value>) -> Decoded> { + match value { + None | Some(Value::Null) => Ok(None), + Some(other) => integer(Some(other)).map(Some), + } +} + +fn path(text: &str) -> Decoded { + normalize_scope_path(text).map_err(|_| Miss) +} + +fn optional_path(value: Option<&Value>) -> Decoded> { + optional_string(value)?.as_deref().map(path).transpose() +} + +fn identity(value: Option<&Value>) -> Decoded { + let item = object(value.ok_or(Miss)?)?; + let kind: String = string(item, "kind")?; + let name: String = string(item, "name")?; + match item.get("type").and_then(Value::as_str) { + Some(RESOURCE_IDENTITY) => Ok(QualifiedIdentity::Resource(ResourceIdentity { + kind: ResourceKind::parse(&kind).ok_or(Miss)?, + name, + })), + Some(DECLARATION_IDENTITY) => { + let owner: Option = match item.get("owner") { + None | Some(Value::Null) => None, + Some(owner) => Some(resource(Some(owner))?), + }; + Ok(QualifiedIdentity::Declaration(DeclarationIdentity { + kind: DeclarationKind::parse(&kind).ok_or(Miss)?, + name, + owner, + })) + } + _ => Err(Miss), + } +} + +fn optional_identity(value: Option<&Value>) -> Decoded> { + match value { + None | Some(Value::Null) => Ok(None), + Some(other) => identity(Some(other)).map(Some), + } +} + +fn resource(value: Option<&Value>) -> Decoded { + match identity(value)? { + QualifiedIdentity::Resource(identity) => Ok(identity), + QualifiedIdentity::Declaration(_) => Err(Miss), + } +} + +fn declaration(value: Option<&Value>) -> Decoded { + match identity(value)? { + QualifiedIdentity::Declaration(identity) => Ok(identity), + QualifiedIdentity::Resource(_) => Err(Miss), + } +} + +fn root(value: Option<&Value>) -> Decoded { + let item = object(value.ok_or(Miss)?)?; + let resource_kind: Option = match item.get("resource_kind") { + Some(Value::String(kind)) => Some(ResourceKind::parse(kind).ok_or(Miss)?), + _ => None, + }; + Ok(OwnershipRoot { + path: path(&string(item, "path")?)?, + kind: OwnershipRootKind::parse(&string(item, "kind")?).ok_or(Miss)?, + resource_kind, + }) +} + +fn macro_facts(value: &Value) -> Decoded { + let item = object(value)?; + let mut dependencies: Vec = Vec::new(); + for raw in list(item, "dependencies")? { + dependencies.push(declaration(Some(raw))?); + } + let mut parameters: Vec = Vec::new(); + for raw in list(item, "parameters")? { + parameters.push(raw.as_str().ok_or(Miss)?.to_owned()); + } + Ok(MacroFacts { + parameters, + dependencies, + source_digest: string(item, "source_digest")?, + }) +} + +fn enum_facts(value: &Value) -> Decoded { + let item = object(value)?; + let mut members: Vec = Vec::new(); + for raw in list(item, "members")? { + let member = object(raw)?; + if member.len() != 1 { + return Err(Miss); + } + members.push(string(member, "name")?); + } + Ok(EnumFacts { + members, + scalar_type: string(item, "scalar_type")?, + }) +} + +fn constant_facts(value: &Value) -> Decoded { + let item = object(value)?; + Ok(ConstantFacts { + logical_type: string(item, "logical_type")?, + collection_kind: optional_string(item.get("collection_kind"))?, + item_count: optional_integer(item.get("item_count"))?, + nullable: boolean(item, "nullable")?, + render_as: optional_string(item.get("render_as"))?, + }) +} + +fn optional(value: Option<&Value>, decode: fn(&Value) -> Decoded) -> Decoded> { + match value { + None | Some(Value::Null) => Ok(None), + Some(other) => decode(other).map(Some), + } +} + +fn cached_declaration(value: &Value) -> Decoded { + let item = object(value)?; + let identity: DeclarationIdentity = declaration(item.get("identity"))?; + let macro_facts: Option = optional(item.get("macro"), macro_facts)?; + let enum_facts: Option = optional(item.get("enum"), enum_facts)?; + let constant_facts: Option = optional(item.get("constant"), constant_facts)?; + Ok(CachedDeclaration { + record: DeclarationRecord { + identity, + path: path(&string(item, "path")?)?, + line: integer(item.get("line"))?, + column: integer(item.get("column"))?, + scope: ScopeKind::parse(&string(item, "scope")?).ok_or(Miss)?, + ownership_root: root(item.get("ownership_root"))?, + owning_path: optional_path(item.get("owning_path"))?, + dependencies: Vec::new(), + }, + macro_facts, + enum_facts, + constant_facts, + }) +} + +fn grant_kind(value: &str) -> Decoded { + [GrantKind::ExpectedModel, GrantKind::TestedMacro] + .into_iter() + .find(|kind| kind.as_str() == value) + .ok_or(Miss) +} + +fn usage(value: &Value) -> Decoded { + let item = object(value)?; + let consumer: QualifiedIdentity = identity(item.get("consumer"))?; + let declared: QualifiedIdentity = identity(item.get("declaration"))?; + let through: Option = optional_identity(item.get("through"))?; + let QualifiedIdentity::Declaration(declaration) = declared else { + return Err(Miss); + }; + Ok(CachedUsage { + consumer, + declaration, + kind: string(item, "kind")?, + through, + enum_member: optional_string(item.get("enum_member"))?, + }) +} + +fn grant(value: &Value) -> Decoded { + let item = object(value)?; + Ok(ReportGrant { + resource: resource(item.get("resource"))?, + declaration: declaration(item.get("declaration"))?, + through: identity(item.get("through"))?, + kind: grant_kind(&string(item, "kind")?)?, + }) +} + +fn visibility(value: &Value) -> Decoded { + let item = object(value)?; + Ok(CachedVisibility { + resource: resource(item.get("resource"))?, + declaration: declaration(item.get("declaration"))?, + reason: string(item, "reason")?, + through: optional_identity(item.get("through"))?, + }) +} + +fn inaccessible(value: &Value) -> Decoded { + let item = object(value)?; + Ok(CachedInaccessible { + resource: resource(item.get("resource"))?, + declaration: declaration(item.get("declaration"))?, + reason: string(item, "reason")?, + }) +} + +fn diagnostic(value: &Value) -> Decoded { + let item = object(value)?; + let declaration: Option = match item.get("declaration") { + None | Some(Value::Null) => None, + Some(other) => Some(declaration(Some(other))?), + }; + let resource: Option = match item.get("resource") { + None | Some(Value::Null) => None, + Some(other) => Some(resource(Some(other))?), + }; + Ok(CachedDiagnostic { + code: string(item, "code")?, + message: string(item, "message")?, + severity: string(item, "severity")?, + path: optional_path(item.get("path"))?, + line: optional_integer(item.get("line"))?, + column: optional_integer(item.get("column"))?, + declaration, + resource, + }) +} + +fn records( + value: &Map, + key: &str, + decode: fn(&Value) -> Decoded, +) -> Decoded> { + list(value, key)?.iter().map(decode).collect() +} + +fn resource_record(value: &Value) -> Decoded { + let item = object(value)?; + Ok(ResourceRecord { + identity: resource(item.get("identity"))?, + path: path(&string(item, "path")?)?, + ownership_root: root(item.get("ownership_root"))?, + }) +} + +fn root_record(value: &Value) -> Decoded { + root(Some(value)) +} + +/// The full index a payload holds, as Python's `decode_scope_index` reads it. +pub(crate) fn decode_index(payload: &Value) -> Decoded { + let value = object(payload)?; + let roots: Vec = records(value, "ownership_roots", root_record)?; + let resources: Vec = records(value, "resources", resource_record)?; + let declarations: Vec = records(value, "declarations", cached_declaration)?; + let usages: Vec = records(value, "usages", usage)?; + let grants: Vec = records(value, "grants", grant)?; + let visibility: Vec = records(value, "visibility", visibility)?; + let inaccessible: Vec = records(value, "inaccessible", inaccessible)?; + let diagnostics: Vec = records(value, "diagnostics", diagnostic)?; + let flags = object(value.get("completeness").ok_or(Miss)?)?; + let mut completeness: [bool; 6] = [false; 6]; + for (slot, key) in completeness.iter_mut().zip(COMPLETENESS_KEYS) { + *slot = boolean(flags, key)?; + } + Ok(CachedIndex { + roots, + resources, + declarations, + usages, + grants, + visibility, + inaccessible, + diagnostics, + completeness, + }) +} + +/// A parsed JSON value as the emitter's value; `Miss` for numbers Python reads differently. +fn json_value(value: &Value) -> Decoded { + Ok(match value { + Value::Null => JsonValue::Null, + Value::Bool(flag) => JsonValue::Bool(*flag), + Value::Number(number) if number.is_i64() || number.is_u64() => { + JsonValue::Integer(JsonInteger::parse(&number.to_string()).ok_or(Miss)?) + } + Value::Number(number) => JsonValue::Float(number.as_f64().ok_or(Miss)?), + Value::String(text) => JsonValue::String(text.clone()), + Value::Array(items) => { + JsonValue::Array(items.iter().map(json_value).collect::>()?) + } + Value::Object(fields) => { + let mut entries: Vec<(String, JsonValue)> = Vec::with_capacity(fields.len()); + for (key, item) in fields { + entries.push((key.clone(), json_value(item)?)); + } + JsonValue::Object(entries) + } + }) +} + +/// The cached index in `raw` for `fingerprint`, as `read_cached_scope_index` checks it. +pub(crate) fn decode_envelope(raw: &[u8], fingerprint: &str) -> Result { + let text: &str = std::str::from_utf8(raw).map_err(|_| CacheMiss::Unreadable)?; + let envelope: Value = serde_json::from_str(text).map_err(|_| CacheMiss::Unreadable)?; + let envelope = envelope.as_object().ok_or(CacheMiss::Stale)?; + let field = + |key: &str| -> Result<&Value, CacheMiss> { envelope.get(key).ok_or(CacheMiss::Stale) }; + if field("schema_version")?.as_i64() != Some(CACHE_SCHEMA_VERSION) + || field("input_fingerprint")?.as_str() != Some(fingerprint) + { + return Err(CacheMiss::Stale); + } + let payload: &Value = field("payload")?; + let canonical: String = json_value(payload) + .map_err(|Miss| CacheMiss::Stale) + .and_then(|value| canonical_json(&value).map_err(|_| CacheMiss::Stale))?; + if field("payload_sha256")?.as_str() != Some(sha256_hex(canonical.as_bytes()).as_str()) { + return Err(CacheMiss::Stale); + } + decode_index(payload).map_err(|Miss| CacheMiss::Invalid) +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/_helpers/fingerprint.rs b/crates/sqlbuild-scopes/src/scope_cache/_helpers/fingerprint.rs new file mode 100644 index 0000000000..158f5b0197 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/_helpers/fingerprint.rs @@ -0,0 +1,183 @@ +//! The fingerprint's view of config files and authored sources. + +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::toml::main::load_toml::load_toml; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; +use sqlbuild_core::posix_path::main::posix_suffix::posix_suffix; +use sqlbuild_core::text::main::active_python_text::active_python_text; + +use std::collections::HashSet; + +use sqlbuild_discovery::tree::main::rglob::rglob; +use sqlbuild_discovery::tree::models::ProjectTree; + +use crate::scope_cache::_helpers::json::{canonical_json, object, sha256_hex}; +use crate::scope_cache::constants::{ + LOCAL_CONFIG_FILENAME, LOCAL_CONFIG_KEYS, MACRO_SUFFIX, PROJECT_CONFIG_KEYS, + RELATIONSHIP_ROOTS, SOURCE_SUFFIXES, TARGET_VARS_KEY, TARGETS_KEY, TOML_SUFFIX, +}; +use crate::scope_cache::models::FingerprintInputs; + +/// A config value `json.dumps` cannot serialize. +pub(crate) struct Unserializable; + +/// Python's `Path(name).suffix` under the running Python. +fn suffix(path: &str) -> &str { + posix_suffix(active_python_text(), path) +} + +/// `_source_role`: declaration, macro, relationship or resource. +pub(crate) fn source_role(relative: &str, inputs: &FingerprintInputs) -> &'static str { + let parts: Vec<&str> = relative.split('/').collect(); + let mut declared: bool = inputs + .named_declaration_directories + .contains(&parts[0].to_owned()); + for part in &parts { + declared |= inputs.declaration_directories.contains(&(*part).to_owned()); + } + if declared { + return "declaration"; + } + if suffix(relative) == MACRO_SUFFIX { + return "macro"; + } + if let [first, second, ..] = parts.as_slice() + && RELATIONSHIP_ROOTS.contains(&(*first, *second)) + { + return "relationship"; + } + "resource" +} + +fn json_value(value: &ConfigValue) -> Result { + Ok(match value { + ConfigValue::Null => JsonValue::Null, + ConfigValue::Bool(flag) => JsonValue::Bool(*flag), + ConfigValue::Integer(number) => { + JsonValue::Integer(JsonInteger::parse(&number.to_string()).ok_or(Unserializable)?) + } + ConfigValue::BigInteger(digits) => { + JsonValue::Integer(JsonInteger::parse(digits).ok_or(Unserializable)?) + } + ConfigValue::Float(number) => JsonValue::Float(*number), + ConfigValue::String(text) => JsonValue::String(text.clone()), + ConfigValue::Date(_) | ConfigValue::DateTime(_) | ConfigValue::Time(_) => { + return Err(Unserializable); + } + ConfigValue::List(items) => { + JsonValue::Array(items.iter().map(json_value).collect::>()?) + } + ConfigValue::Map(entries) => { + let mut fields: Vec<(String, JsonValue)> = Vec::with_capacity(entries.len()); + for (key, item) in entries { + let ConfigValue::String(key) = key else { + return Err(Unserializable); + }; + fields.push((key.clone(), json_value(item)?)); + } + JsonValue::Object(fields) + } + }) +} + +/// `_scope_target_configs`: each named target's vars. +fn target_vars(value: &ConfigValue) -> Result { + let Some(entries) = value.as_map() else { + return Ok(JsonValue::Object(Vec::new())); + }; + let mut targets: Vec<(String, JsonValue)> = Vec::new(); + for (name, config) in entries { + let (ConfigValue::String(name), Some(_)) = (name, config.as_map()) else { + continue; + }; + let vars: JsonValue = match config.get(TARGET_VARS_KEY) { + Some(vars) => json_value(vars)?, + None => JsonValue::Object(Vec::new()), + }; + targets.push((name.clone(), object(vec![(TARGET_VARS_KEY, vars)]))); + } + Ok(JsonValue::Object(targets)) +} + +/// `_scope_config_bytes`: a TOML config's scope-relevant keys as canonical JSON, else `raw`. +pub(crate) fn config_bytes(name: &str, raw: &[u8]) -> Result, Unserializable> { + if !name.ends_with(TOML_SUFFIX) { + return Ok(raw.to_vec()); + } + let Ok(text) = std::str::from_utf8(raw) else { + return Ok(raw.to_vec()); + }; + let Ok(document) = load_toml(text) else { + return Ok(raw.to_vec()); + }; + let Some(entries) = document.as_map() else { + return Ok(raw.to_vec()); + }; + let keys: &[&str] = if name == LOCAL_CONFIG_FILENAME { + &LOCAL_CONFIG_KEYS + } else { + &PROJECT_CONFIG_KEYS + }; + let mut relevant: Vec<(String, JsonValue)> = Vec::new(); + for (key, value) in entries { + let ConfigValue::String(key) = key else { + continue; + }; + if !keys.contains(&key.as_str()) { + continue; + } + let value: JsonValue = if key == TARGETS_KEY { + target_vars(value)? + } else { + json_value(value)? + }; + relevant.push((key.clone(), value)); + } + canonical_json(&JsonValue::Object(relevant)) + .map(String::into_bytes) + .map_err(|_| Unserializable) +} + +/// `(path, role, content hash)` of one hashed source. +pub(crate) type SourceRecord = (String, &'static str, String); + +pub(crate) fn source_record(path: &str, role: &'static str, content: &[u8]) -> SourceRecord { + (path.replace('\\', "/"), role, sha256_hex(content)) +} + +/// Every hashable authored source below `roots`, unsorted; `None` when one cannot be read. +pub(crate) fn source_records( + project_dir: &std::path::Path, + roots: &[String], + inputs: &FingerprintInputs, +) -> Option> { + let tree: ProjectTree = ProjectTree::new(project_dir); + let mut seen: HashSet = HashSet::new(); + let mut records: Vec = Vec::new(); + for root in roots { + let Ok(paths) = rglob(&tree, root, |entry| !entry.is_dir) else { + return None; + }; + for relative in paths { + if tree.is_undecodable(&relative) { + return None; + } + let absolute = tree.absolute(&relative); + if !absolute.is_file() + || !SOURCE_SUFFIXES.contains(&suffix(&relative).to_lowercase().as_str()) + || !seen.insert(relative.clone()) + { + continue; + } + let Ok(content) = std::fs::read(&absolute) else { + return None; + }; + records.push(source_record( + &relative, + source_role(&relative, inputs), + &content, + )); + } + } + Some(records) +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/_helpers/json.rs b/crates/sqlbuild-scopes/src/scope_cache/_helpers/json.rs new file mode 100644 index 0000000000..3a2322d4c5 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/_helpers/json.rs @@ -0,0 +1,51 @@ +//! Python's canonical cache JSON (`ensure_ascii`, compact separators, sorted keys) and SHA-256. + +use sha2::{Digest, Sha256}; +use sqlbuild_core::json::errors::JsonEmitError; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +/// The canonical JSON text, or why Python's encoder raises. +pub(crate) fn canonical_json(value: &JsonValue) -> Result { + let mut options: StdlibJsonOptions = StdlibJsonOptions::new(None).with_separators(",", ":"); + options.sort_keys = true; + dumps(value, &JsonDialect::Stdlib(options)) +} + +/// The lowercase hex SHA-256 of `bytes`. +pub(crate) fn sha256_hex(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut text: String = String::with_capacity(digest.len() * 2); + for byte in digest { + text.push_str(&format!("{byte:02x}")); + } + text +} + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(crate) fn optional_text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, text) +} + +pub(crate) fn integer(value: i64) -> JsonValue { + match JsonInteger::parse(&value.to_string()) { + Some(number) => JsonValue::Integer(number), + None => JsonValue::Null, + } +} + +pub(crate) fn optional_integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, integer) +} + +pub(crate) fn object(fields: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + fields + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/_helpers/mod.rs b/crates/sqlbuild-scopes/src/scope_cache/_helpers/mod.rs new file mode 100644 index 0000000000..f7c20a894b --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod decode; +pub(crate) mod fingerprint; +pub(crate) mod json; +pub(crate) mod payload; diff --git a/crates/sqlbuild-scopes/src/scope_cache/_helpers/payload.rs b/crates/sqlbuild-scopes/src/scope_cache/_helpers/payload.rs new file mode 100644 index 0000000000..da4b62d2c3 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/_helpers/payload.rs @@ -0,0 +1,297 @@ +//! The cache payload of a full scope index in the lookup's canonical order. + +use sqlbuild_core::json::models::JsonValue; + +use crate::scope_cache::_helpers::json::{integer, object, optional_integer, optional_text, text}; +use crate::scope_cache::constants::{COMPLETENESS_KEYS, DECLARATION_IDENTITY, RESOURCE_IDENTITY}; +use crate::scope_cache::models::{CachedDeclaration, CachedIndex}; +use crate::scope_index::main::format_identity::format_identity; +use crate::scope_index::main::normalize_scope_path::normalize_scope_path; +use crate::scope_index::models::{ + DeclarationIdentity, OwnershipRoot, QualifiedIdentity, ReportLookup, ResourceIdentity, + ResourceKind, +}; + +/// A path the payload cannot normalize, which makes Python skip the cache write. +pub(crate) struct Unwritable; + +fn path(value: &str) -> Result { + normalize_scope_path(value) + .map(JsonValue::String) + .map_err(|_| Unwritable) +} + +fn optional_path(value: Option<&str>) -> Result { + value.map_or(Ok(JsonValue::Null), path) +} + +pub(crate) fn resource_identity(identity: &ResourceIdentity) -> JsonValue { + object(vec![ + ("type", text(RESOURCE_IDENTITY)), + ("kind", text(identity.kind.as_str())), + ("name", text(&identity.name)), + ]) +} + +pub(crate) fn declaration_identity(identity: &DeclarationIdentity) -> JsonValue { + object(vec![ + ("type", text(DECLARATION_IDENTITY)), + ("kind", text(identity.kind.as_str())), + ("name", text(&identity.name)), + ( + "owner", + identity + .owner + .as_ref() + .map_or(JsonValue::Null, resource_identity), + ), + ]) +} + +pub(crate) fn qualified_identity(identity: &QualifiedIdentity) -> JsonValue { + match identity { + QualifiedIdentity::Resource(identity) => resource_identity(identity), + QualifiedIdentity::Declaration(identity) => declaration_identity(identity), + } +} + +fn optional_identity(identity: Option<&QualifiedIdentity>) -> JsonValue { + identity.map_or(JsonValue::Null, qualified_identity) +} + +fn root(root: &OwnershipRoot) -> Result { + Ok(object(vec![ + ("path", path(&root.path)?), + ("kind", text(root.kind.as_str())), + ( + "resource_kind", + optional_text(root.resource_kind.map(ResourceKind::as_str)), + ), + ])) +} + +fn declaration(item: &CachedDeclaration) -> Result { + let record = &item.record; + let macro_facts: JsonValue = item.macro_facts.as_ref().map_or(JsonValue::Null, |facts| { + object(vec![ + ( + "parameters", + JsonValue::Array(facts.parameters.iter().map(|value| text(value)).collect()), + ), + ( + "dependencies", + JsonValue::Array( + facts + .dependencies + .iter() + .map(declaration_identity) + .collect(), + ), + ), + ("source_digest", text(&facts.source_digest)), + ]) + }); + let enum_facts: JsonValue = item.enum_facts.as_ref().map_or(JsonValue::Null, |facts| { + object(vec![ + ( + "members", + JsonValue::Array( + facts + .members + .iter() + .map(|name| object(vec![("name", text(name))])) + .collect(), + ), + ), + ("scalar_type", text(&facts.scalar_type)), + ]) + }); + let constant_facts: JsonValue = item + .constant_facts + .as_ref() + .map_or(JsonValue::Null, |facts| { + object(vec![ + ("logical_type", text(&facts.logical_type)), + ( + "collection_kind", + optional_text(facts.collection_kind.as_deref()), + ), + ("item_count", optional_integer(facts.item_count)), + ("nullable", JsonValue::Bool(facts.nullable)), + ("render_as", optional_text(facts.render_as.as_deref())), + ]) + }); + Ok(object(vec![ + ("identity", declaration_identity(&record.identity)), + ("path", path(&record.path)?), + ("line", integer(record.line)), + ("column", integer(record.column)), + ("scope", text(record.scope.as_str())), + ("ownership_root", root(&record.ownership_root)?), + ("owning_path", optional_path(record.owning_path.as_deref())?), + ("macro", macro_facts), + ("enum", enum_facts), + ("constant", constant_facts), + ])) +} + +fn resource_text(identity: &ResourceIdentity) -> String { + format_identity(&QualifiedIdentity::Resource(identity.clone())) +} + +fn declaration_text(identity: &DeclarationIdentity) -> String { + format_identity(&QualifiedIdentity::Declaration(identity.clone())) +} + +/// Stable positions of `count` items ordered by `key`. +fn ordered(count: usize, key: impl Fn(usize) -> K) -> Vec { + let mut positions: Vec = (0..count).collect(); + positions.sort_by_cached_key(|position| key(*position)); + positions +} + +fn array( + order: &[usize], + item: impl Fn(usize) -> Result, +) -> Result { + Ok(JsonValue::Array( + order + .iter() + .map(|position| item(*position)) + .collect::>()?, + )) +} + +/// The payload object Python's `_index_payload` builds for the canonical index. +pub(crate) fn index_payload( + index: &CachedIndex, + lookup: &ReportLookup, +) -> Result { + let visibility: Vec = ordered(index.visibility.len(), |position| { + let item = &index.visibility[position]; + ( + resource_text(&item.resource), + declaration_text(&item.declaration), + item.reason.clone(), + item.through + .as_ref() + .map(format_identity) + .unwrap_or_default(), + ) + }); + let inaccessible: Vec = ordered(index.inaccessible.len(), |position| { + let item = &index.inaccessible[position]; + ( + resource_text(&item.resource), + declaration_text(&item.declaration), + item.reason.clone(), + ) + }); + Ok(object(vec![ + ( + "ownership_roots", + array(&lookup.roots, |position| root(&index.roots[position]))?, + ), + ( + "resources", + array(&lookup.resources, |position| { + let record = &index.resources[position]; + Ok(object(vec![ + ("identity", resource_identity(&record.identity)), + ("path", path(&record.path)?), + ("ownership_root", root(&record.ownership_root)?), + ])) + })?, + ), + ( + "declarations", + array(&lookup.declarations, |position| { + declaration(&index.declarations[position]) + })?, + ), + ( + "usages", + array(&lookup.usages, |position| { + let item = &index.usages[position]; + Ok(object(vec![ + ("consumer", qualified_identity(&item.consumer)), + ("declaration", declaration_identity(&item.declaration)), + ("kind", text(&item.kind)), + ("through", optional_identity(item.through.as_ref())), + ("enum_member", optional_text(item.enum_member.as_deref())), + ])) + })?, + ), + ( + "grants", + array(&lookup.grants, |position| { + let item = &index.grants[position]; + Ok(object(vec![ + ("resource", resource_identity(&item.resource)), + ("declaration", declaration_identity(&item.declaration)), + ("through", qualified_identity(&item.through)), + ("kind", text(item.kind.as_str())), + ])) + })?, + ), + ( + "visibility", + array(&visibility, |position| { + let item = &index.visibility[position]; + Ok(object(vec![ + ("resource", resource_identity(&item.resource)), + ("declaration", declaration_identity(&item.declaration)), + ("reason", text(&item.reason)), + ("through", optional_identity(item.through.as_ref())), + ])) + })?, + ), + ( + "inaccessible", + array(&inaccessible, |position| { + let item = &index.inaccessible[position]; + Ok(object(vec![ + ("resource", resource_identity(&item.resource)), + ("declaration", declaration_identity(&item.declaration)), + ("reason", text(&item.reason)), + ])) + })?, + ), + ( + "diagnostics", + array(&lookup.diagnostics, |position| { + let item = &index.diagnostics[position]; + Ok(object(vec![ + ("code", text(&item.code)), + ("message", text(&item.message)), + ("severity", text(&item.severity)), + ("path", optional_path(item.path.as_deref())?), + ("line", optional_integer(item.line)), + ("column", optional_integer(item.column)), + ( + "declaration", + item.declaration + .as_ref() + .map_or(JsonValue::Null, declaration_identity), + ), + ( + "resource", + item.resource + .as_ref() + .map_or(JsonValue::Null, resource_identity), + ), + ])) + })?, + ), + ( + "completeness", + object( + COMPLETENESS_KEYS + .iter() + .zip(index.completeness) + .map(|(key, complete)| (*key, JsonValue::Bool(complete))) + .collect(), + ), + ), + ])) +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/constants.rs b/crates/sqlbuild-scopes/src/scope_cache/constants.rs new file mode 100644 index 0000000000..a0069851c9 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/constants.rs @@ -0,0 +1,38 @@ +//! Cache schema and fingerprint versions, keys and the scope-relevant source facts. + +pub(crate) const CACHE_SCHEMA_VERSION: i64 = 3; +pub(crate) const FINGERPRINT_ALGORITHM_VERSION: i64 = 2; +/// The largest cache file read; bigger files are misses. +pub(crate) const CACHE_MAX_BYTES: u64 = 16 * 1024 * 1024; +pub(crate) const RESOURCE_IDENTITY: &str = "resource"; +pub(crate) const DECLARATION_IDENTITY: &str = "declaration"; +/// Authored suffixes, compared lowercased, whose contents feed the fingerprint. +pub(crate) const SOURCE_SUFFIXES: [&str; 4] = [".sql", ".yml", ".yaml", ".py"]; +pub(crate) const MACRO_SUFFIX: &str = ".py"; +pub(crate) const RELATIONSHIP_ROOTS: [(&str, &str); 2] = + [("tests", "unit"), ("tests", "scenarios")]; +pub(crate) const LOCAL_CONFIG_KEYS: [&str; 5] = + ["adapter", "settings", "target", "targets", "vars"]; +pub(crate) const PROJECT_CONFIG_KEYS: [&str; 9] = [ + "adapter", + "constants", + "default_target", + "defaults", + "path_defaults", + "scopes", + "settings", + "targets", + "vars", +]; +pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +pub(crate) const TARGETS_KEY: &str = "targets"; +pub(crate) const TARGET_VARS_KEY: &str = "vars"; +pub(crate) const TOML_SUFFIX: &str = ".toml"; +pub(crate) const COMPLETENESS_KEYS: [&str; 6] = [ + "discovery", + "static_visibility", + "runtime_usage", + "relationships", + "placement", + "promotion_impact", +]; diff --git a/crates/sqlbuild-scopes/src/scope_cache/main/encode_scope_cache.rs b/crates/sqlbuild-scopes/src/scope_cache/main/encode_scope_cache.rs new file mode 100644 index 0000000000..40db9988ad --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/main/encode_scope_cache.rs @@ -0,0 +1,57 @@ +//! `write_cached_scope_index`'s envelope of one full scope index. + +use sqlbuild_core::json::models::JsonValue; + +use crate::scope_cache::_helpers::json::{canonical_json, integer, object, sha256_hex, text}; +use crate::scope_cache::_helpers::payload::{Unwritable, index_payload}; +use crate::scope_cache::constants::CACHE_SCHEMA_VERSION; +use crate::scope_cache::models::{CachedIndex, Unencodable}; +use crate::scope_index::main::report_lookup::report_lookup; +use crate::scope_index::models::{ReportDiagnosticKey, ReportLookup, ReportRecords, ReportUsage}; + +/// The canonical envelope bytes, or `Unencodable` when Python would skip the write. +pub fn encode_scope_cache(index: &CachedIndex, fingerprint: &str) -> Result, Unencodable> { + let records: ReportRecords = ReportRecords { + roots: index.roots.clone(), + resources: index.resources.clone(), + declarations: index + .declarations + .iter() + .map(|item| item.record.clone()) + .collect(), + usages: index + .usages + .iter() + .map(|item| ReportUsage { + consumer: item.consumer.clone(), + declaration: item.declaration.clone(), + kind: item.kind.clone(), + through: item.through.clone(), + }) + .collect(), + grants: index.grants.clone(), + diagnostics: index + .diagnostics + .iter() + .map(|item| ReportDiagnosticKey { + path: item.path.clone(), + line: item.line, + column: item.column, + code: item.code.clone(), + }) + .collect(), + }; + let lookup: ReportLookup = report_lookup(&records).map_err(|_| Unencodable)?; + let payload: JsonValue = index_payload(index, &lookup).map_err(|Unwritable| Unencodable)?; + let encoded: String = canonical_json(&payload).map_err(|_| Unencodable)?; + let digest: String = sha256_hex(encoded.as_bytes()); + let envelope: JsonValue = object(vec![ + ("input_fingerprint", text(fingerprint)), + ("payload", payload), + ("payload_sha256", text(&digest)), + ("schema_version", integer(CACHE_SCHEMA_VERSION)), + ]); + canonical_json(&envelope) + .map(String::into_bytes) + .map_err(|_| Unencodable) +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/main/mod.rs b/crates/sqlbuild-scopes/src/scope_cache/main/mod.rs new file mode 100644 index 0000000000..e5718a1fb4 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/main/mod.rs @@ -0,0 +1,4 @@ +pub mod encode_scope_cache; +pub mod read_scope_cache; +pub mod scope_fingerprint; +pub mod write_scope_cache; diff --git a/crates/sqlbuild-scopes/src/scope_cache/main/read_scope_cache.rs b/crates/sqlbuild-scopes/src/scope_cache/main/read_scope_cache.rs new file mode 100644 index 0000000000..cdd0b77211 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/main/read_scope_cache.rs @@ -0,0 +1,19 @@ +//! `read_cached_scope_index`: every storage or validation fault is a miss. + +use std::path::Path; + +use crate::scope_cache::_helpers::decode::decode_envelope; +use crate::scope_cache::constants::CACHE_MAX_BYTES; +use crate::scope_cache::models::{CacheMiss, CachedIndex}; + +/// The verified cached index at `path` for `fingerprint`, else why it is a miss. +pub fn read_scope_cache(path: &Path, fingerprint: &str) -> Result { + let size: u64 = std::fs::metadata(path) + .map_err(|_| CacheMiss::Unreadable)? + .len(); + if size > CACHE_MAX_BYTES { + return Err(CacheMiss::Unreadable); + } + let raw: Vec = std::fs::read(path).map_err(|_| CacheMiss::Unreadable)?; + decode_envelope(&raw, fingerprint) +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/main/scope_fingerprint.rs b/crates/sqlbuild-scopes/src/scope_cache/main/scope_fingerprint.rs new file mode 100644 index 0000000000..e5d7578db2 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/main/scope_fingerprint.rs @@ -0,0 +1,61 @@ +//! `scope_index_fingerprint`: a hash of the scope-relevant config and authored sources. + +use std::path::Path; + +use sqlbuild_core::json::models::JsonValue; + +use crate::scope_cache::_helpers::fingerprint::{ + SourceRecord, config_bytes, source_record, source_records, +}; +use crate::scope_cache::_helpers::json::{canonical_json, integer, object, sha256_hex, text}; +use crate::scope_cache::constants::{CACHE_SCHEMA_VERSION, FINGERPRINT_ALGORITHM_VERSION}; +use crate::scope_cache::models::{Fingerprint, FingerprintInputs}; + +/// The fingerprint of `project_dir`, or why Python has none or raises. +#[must_use] +pub fn scope_fingerprint(project_dir: &Path, inputs: &FingerprintInputs) -> Fingerprint { + let mut records: Vec = Vec::new(); + for name in &inputs.config_files { + let path = project_dir.join(name); + if !path.is_file() { + continue; + } + let Ok(raw) = std::fs::read(&path) else { + return Fingerprint::Unavailable; + }; + let Ok(content) = config_bytes(name, &raw) else { + return Fingerprint::Unserializable; + }; + records.push(source_record(name, "config", &content)); + } + let mut roots: Vec = inputs.roots.clone(); + roots.sort(); + roots.dedup(); + let Some(sources) = source_records(project_dir, &roots, inputs) else { + return Fingerprint::Unavailable; + }; + records.extend(sources); + records.sort_by(|left, right| (&left.0, left.1).cmp(&(&right.0, right.1))); + let mut sources: Vec = Vec::with_capacity(records.len()); + for (path, role, digest) in &records { + sources.push(object(vec![ + ("path", text(path)), + ("role", text(role)), + ("sha256", text(digest)), + ])); + } + let payload: JsonValue = object(vec![ + ("algorithm_version", integer(FINGERPRINT_ALGORITHM_VERSION)), + ("cache_schema_version", integer(CACHE_SCHEMA_VERSION)), + ( + "ownership_roots", + JsonValue::Array(roots.iter().map(|root| text(root)).collect()), + ), + ("sources", JsonValue::Array(sources)), + ("sqlbuild_version", text(&inputs.sqlbuild_version)), + ]); + match canonical_json(&payload) { + Ok(encoded) => Fingerprint::Hash(sha256_hex(encoded.as_bytes())), + Err(_) => Fingerprint::Unavailable, + } +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/main/write_scope_cache.rs b/crates/sqlbuild-scopes/src/scope_cache/main/write_scope_cache.rs new file mode 100644 index 0000000000..05c7c184cb --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/main/write_scope_cache.rs @@ -0,0 +1,26 @@ +//! `write_cached_scope_index`'s atomic replace, ignoring I/O faults. + +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Write `envelope` to `directory/file_name` through a synced temporary file. +pub fn write_scope_cache(directory: &Path, file_name: &str, envelope: &[u8]) -> io::Result<()> { + std::fs::create_dir_all(directory)?; + let nonce: u128 = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos()); + let temporary: PathBuf = + directory.join(format!(".scope-index-{}-{nonce:x}", std::process::id())); + let written: io::Result<()> = std::fs::File::create_new(&temporary) + .and_then(|mut stream| { + stream.write_all(envelope)?; + stream.flush()?; + stream.sync_all() + }) + .and_then(|()| std::fs::rename(&temporary, directory.join(file_name))); + if written.is_err() { + let _ = std::fs::remove_file(&temporary); + } + written +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/mod.rs b/crates/sqlbuild-scopes/src/scope_cache/mod.rs new file mode 100644 index 0000000000..93d6cf55c8 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/mod.rs @@ -0,0 +1,9 @@ +//! The persistent scope index cache: its canonical JSON codec and input fingerprint. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-scopes/src/scope_cache/models.rs b/crates/sqlbuild-scopes/src/scope_cache/models.rs new file mode 100644 index 0000000000..c2b01f8b80 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/models.rs @@ -0,0 +1,134 @@ +//! A full scope index as the cache stores it, and the fingerprint's inputs and outcome. + +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, OwnershipRoot, QualifiedIdentity, ReportGrant, + ResourceIdentity, ResourceRecord, +}; + +/// A macro declaration's parameters, dependencies and source digest. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MacroFacts { + pub parameters: Vec, + pub dependencies: Vec, + pub source_digest: String, +} + +/// An enum declaration's member names and scalar type. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EnumFacts { + pub members: Vec, + pub scalar_type: String, +} + +/// A constant declaration's value-free shape. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConstantFacts { + pub logical_type: String, + pub collection_kind: Option, + pub item_count: Option, + pub nullable: bool, + pub render_as: Option, +} + +/// A declaration record with its value metadata. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CachedDeclaration { + pub record: DeclarationRecord, + pub macro_facts: Option, + pub enum_facts: Option, + pub constant_facts: Option, +} + +/// One usage with its kind text and the enum member it reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CachedUsage { + pub consumer: QualifiedIdentity, + pub declaration: DeclarationIdentity, + pub kind: String, + pub through: Option, + pub enum_member: Option, +} + +/// One precomputed visibility fact with its reason text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CachedVisibility { + pub resource: ResourceIdentity, + pub declaration: DeclarationIdentity, + pub reason: String, + pub through: Option, +} + +/// One precomputed inaccessible fact with its reason text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CachedInaccessible { + pub resource: ResourceIdentity, + pub declaration: DeclarationIdentity, + pub reason: String, +} + +/// One scope diagnostic with its code and severity text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CachedDiagnostic { + pub code: String, + pub message: String, + pub severity: String, + pub path: Option, + pub line: Option, + pub column: Option, + pub declaration: Option, + pub resource: Option, +} + +/// Python's `ScopeIndex` with every field the cache stores. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CachedIndex { + pub roots: Vec, + pub resources: Vec, + pub declarations: Vec, + pub usages: Vec, + pub grants: Vec, + pub visibility: Vec, + pub inaccessible: Vec, + pub diagnostics: Vec, + /// Discovery, static visibility, runtime usage, relationships, placement, promotion impact. + pub completeness: [bool; 6], +} + +/// What the fingerprint reads besides the project's files. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FingerprintInputs { + /// Config files hashed when present, in order. + pub config_files: Vec, + /// Every candidate source root; the fingerprint hashes their sorted distinct set. + pub roots: Vec, + /// Folder names that make any file below them a declaration source. + pub declaration_directories: Vec, + /// First-level folders whose files are named declaration sources. + pub named_declaration_directories: Vec, + pub sqlbuild_version: String, +} + +/// The fingerprint, or why Python would have no fingerprint or raise. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Fingerprint { + Hash(String), + /// An I/O, decoding or value fault: Python's caught `OSError`, `UnicodeError` or `ValueError`. + Unavailable, + /// A config value `json.dumps` cannot serialize: Python's uncaught `TypeError`. + Unserializable, +} + +/// Why a cache file is a miss. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CacheMiss { + /// Unreadable, oversized or not JSON. + Unreadable, + /// The envelope's schema, fingerprint or payload digest does not match. + Stale, + /// The payload fails Python's decoder. + Invalid, +} + +/// Why a full index has no cache envelope: Python skips the write. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Unencodable; diff --git a/crates/sqlbuild-scopes/src/scope_cache/tests.rs b/crates/sqlbuild-scopes/src/scope_cache/tests.rs new file mode 100644 index 0000000000..fd6470b690 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_scope_cache.rs"] +mod scope_cache; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-scopes/src/scope_cache/tests/helpers.rs b/crates/sqlbuild-scopes/src/scope_cache/tests/helpers.rs new file mode 100644 index 0000000000..d9a5e1e91c --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/tests/helpers.rs @@ -0,0 +1,70 @@ +use crate::scope_cache::models::{CachedDeclaration, CachedIndex, EnumFacts}; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationKind, DeclarationRecord, OwnershipRoot, OwnershipRootKind, + ResourceIdentity, ResourceKind, ResourceRecord, ScopeKind, +}; + +fn models_root() -> OwnershipRoot { + OwnershipRoot { + path: "models".to_owned(), + kind: OwnershipRootKind::Resource, + resource_kind: Some(ResourceKind::Model), + } +} + +/// A complete index with one model and one enum declaration. +pub(super) fn orders_index() -> CachedIndex { + CachedIndex { + roots: vec![models_root()], + resources: vec![ResourceRecord { + identity: ResourceIdentity { + kind: ResourceKind::Model, + name: "orders".to_owned(), + }, + path: "models/sales/orders.sql".to_owned(), + ownership_root: models_root(), + }], + declarations: vec![CachedDeclaration { + record: DeclarationRecord { + identity: DeclarationIdentity { + kind: DeclarationKind::Enum, + name: "order_status".to_owned(), + owner: None, + }, + path: "models/sales/enums/status.yml".to_owned(), + line: 1, + column: 1, + scope: ScopeKind::Inherited, + ownership_root: models_root(), + owning_path: Some("models/sales".to_owned()), + dependencies: Vec::new(), + }, + macro_facts: None, + enum_facts: Some(EnumFacts { + members: vec!["placed".to_owned(), "shipped".to_owned()], + scalar_type: "VARCHAR".to_owned(), + }), + constant_facts: None, + }], + completeness: [true; 6], + ..CachedIndex::default() + } +} + +pub(super) fn unchanged(bytes: Vec) -> Vec { + bytes +} + +/// The envelope with its payload digest's first hex digit replaced by one that differs. +pub(super) fn wrong_digest(bytes: Vec) -> Vec { + let text: String = String::from_utf8(bytes).expect("ascii envelope"); + let marker: &str = "\"payload_sha256\":\""; + let start: usize = text.find(marker).expect("digest field") + marker.len(); + let digit: char = text[start..].chars().next().expect("digest digit"); + let replacement: &str = ["0", "1"][usize::from(digit == '0')]; + format!("{}{replacement}{}", &text[..start], &text[start + 1..]).into_bytes() +} + +pub(super) fn truncated(bytes: Vec) -> Vec { + bytes[..bytes.len() / 2].to_vec() +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/tests/test_scope_cache.rs b/crates/sqlbuild-scopes/src/scope_cache/tests/test_scope_cache.rs new file mode 100644 index 0000000000..b7424a486f --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/tests/test_scope_cache.rs @@ -0,0 +1,84 @@ +use crate::scope_cache::_helpers::decode::decode_envelope; +use crate::scope_cache::_helpers::fingerprint::config_bytes; +use crate::scope_cache::main::encode_scope_cache::encode_scope_cache; +use crate::scope_cache::models::{CacheMiss, CachedIndex}; +use crate::scope_cache::tests::helpers::{orders_index, truncated, unchanged, wrong_digest}; +use crate::scope_cache::tests::test_types::{ConfigBytesTestCase, ReadBackTestCase}; + +#[test] +fn given_written_envelope_when_reading_back_then_only_an_intact_matching_cache_hits() { + let test_cases = [ + ReadBackTestCase { + description: "an intact envelope for the same fingerprint decodes the same index", + read_fingerprint: "inputs", + altered: unchanged, + expected_read: Ok(()), + }, + ReadBackTestCase { + description: "another fingerprint is stale", + read_fingerprint: "other", + altered: unchanged, + expected_read: Err(CacheMiss::Stale), + }, + ReadBackTestCase { + description: "a payload digest mismatch is stale", + read_fingerprint: "inputs", + altered: wrong_digest, + expected_read: Err(CacheMiss::Stale), + }, + ReadBackTestCase { + description: "truncated bytes are unreadable", + read_fingerprint: "inputs", + altered: truncated, + expected_read: Err(CacheMiss::Unreadable), + }, + ]; + for test_case in test_cases { + let envelope: Vec = encode_scope_cache(&orders_index(), "inputs").expect("encodes"); + + let read: Result = + decode_envelope(&(test_case.altered)(envelope), test_case.read_fingerprint); + + assert_eq!( + read.map(|index| index == orders_index()), + test_case.expected_read.map(|()| true), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_config_file_when_fingerprinting_then_only_scope_relevant_keys_are_hashed() { + let test_cases = [ + ConfigBytesTestCase { + description: "keys outside the scope set and target fields beyond vars drop", + name: "sqlbuild_project.toml", + text: "name = \"orders\"\n[vars]\nregion = \"eu\"\n[targets.dev]\nschema = \"x\"\n[targets.dev.vars]\nday = 1\n", + expected_bytes: "{\"targets\":{\"dev\":{\"vars\":{\"day\":1}}},\"vars\":{\"region\":\"eu\"}}", + }, + ConfigBytesTestCase { + description: "a non-TOML config file hashes its raw bytes", + name: ".sqlbuildignore", + text: "models/tmp\n", + expected_bytes: "models/tmp\n", + }, + ConfigBytesTestCase { + description: "invalid TOML hashes its raw bytes", + name: "sqlbuild_local.toml", + text: "adapter = \n", + expected_bytes: "adapter = \n", + }, + ]; + for test_case in test_cases { + let bytes: Vec = config_bytes(test_case.name, test_case.text.as_bytes()) + .unwrap_or_else(|_| panic!("{} is serializable", test_case.description)); + + assert_eq!( + String::from_utf8(bytes).expect("utf-8"), + test_case.expected_bytes, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_cache/tests/test_types.rs b/crates/sqlbuild-scopes/src/scope_cache/tests/test_types.rs new file mode 100644 index 0000000000..6b0e50a1fd --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_cache/tests/test_types.rs @@ -0,0 +1,17 @@ +use crate::scope_cache::models::CacheMiss; + +/// A written envelope, how its bytes are altered, and what reading it back gives. +pub(super) struct ReadBackTestCase { + pub(super) description: &'static str, + pub(super) read_fingerprint: &'static str, + pub(super) altered: fn(Vec) -> Vec, + pub(super) expected_read: Result<(), CacheMiss>, +} + +/// A config file name and text, and the bytes the fingerprint hashes for it. +pub(super) struct ConfigBytesTestCase { + pub(super) description: &'static str, + pub(super) name: &'static str, + pub(super) text: &'static str, + pub(super) expected_bytes: &'static str, +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/identities.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/identities.rs index 953352e6d7..22278aa28b 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/identities.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/identities.rs @@ -1,8 +1,7 @@ //! Qualified identity text (`format_identity`) and the `repr` text Python orders lookup keys by. -use crate::scope_index::models::{ - DeclarationIdentity, GrantRecord, GrantThrough, ResourceIdentity, ScopeIndex, -}; +use crate::scope_index::models::{DeclarationIdentity, QualifiedIdentity, ResourceIdentity}; +use sqlbuild_core::text::main::active_python_text::active_python_text; use sqlbuild_core::text::main::python_str_repr::python_str_repr; /// Python's `format_identity` for a resource. @@ -24,36 +23,56 @@ pub(crate) fn declaration_text(identity: &DeclarationIdentity) -> String { } } -/// Python's `format_identity` of what a grant came through. -pub(crate) fn grant_through_text(index: &ScopeIndex, grant: &GrantRecord) -> String { - match &grant.through { - GrantThrough::Model(name) => format!("model:{name}"), - GrantThrough::Declaration(declaration) => { - declaration_text(&index.declarations[*declaration].identity) - } - } -} - /// Python's dataclass `repr` of a `ResourceIdentity`. -pub(crate) fn resource_repr(identity: &ResourceIdentity) -> Option { - Some(format!( +pub(crate) fn resource_repr(identity: &ResourceIdentity) -> String { + format!( "ResourceIdentity(kind=, name={})", identity.kind.member_name(), identity.kind.as_str(), - python_str_repr(&identity.name)? - )) + python_str_repr(active_python_text(), &identity.name) + ) } /// Python's dataclass `repr` of a `DeclarationIdentity`. -pub(crate) fn declaration_repr(identity: &DeclarationIdentity) -> Option { +pub(crate) fn declaration_repr(identity: &DeclarationIdentity) -> String { let owner: String = match &identity.owner { None => "None".to_owned(), - Some(owner) => resource_repr(owner)?, + Some(owner) => resource_repr(owner), }; - Some(format!( + format!( "DeclarationIdentity(kind=, name={}, owner={owner})", identity.kind.member_name(), identity.kind.as_str(), - python_str_repr(&identity.name)? - )) + python_str_repr(active_python_text(), &identity.name) + ) +} + +/// Python's `format_identity` of a resource or declaration. +pub(crate) fn qualified_text(identity: &QualifiedIdentity) -> String { + match identity { + QualifiedIdentity::Resource(identity) => resource_text(identity), + QualifiedIdentity::Declaration(identity) => declaration_text(identity), + } +} + +/// Python's dataclass `repr` of a resource or declaration identity. +pub(crate) fn qualified_repr(identity: &QualifiedIdentity) -> String { + match identity { + QualifiedIdentity::Resource(identity) => resource_repr(identity), + QualifiedIdentity::Declaration(identity) => declaration_repr(identity), + } +} + +/// Python's `format_identity` of what a static index grant came through. +#[cfg(test)] +pub(crate) fn grant_through_text( + index: &crate::scope_index::models::ScopeIndex, + grant: &crate::scope_index::models::GrantRecord, +) -> String { + match &grant.through { + crate::scope_index::models::GrantThrough::Model(name) => format!("model:{name}"), + crate::scope_index::models::GrantThrough::Declaration(declaration) => { + declaration_text(&index.declarations[*declaration].identity) + } + } } diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs index f9c566c15a..0a9859764e 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs @@ -76,24 +76,19 @@ pub(crate) fn group_in_order( groups } -/// Python's `group_records`: groups ordered by `repr(key)` when every key's repr is known. +/// Python's `group_records`: groups ordered by `repr(key)`, members in input order. pub(crate) fn keyed_groups( members: &[usize], key: impl Fn(usize) -> K, - repr: impl Fn(&K) -> Option, + repr: impl Fn(&K) -> String, ) -> KeyedGroups { let groups: Vec> = group_in_order(members, &key); - let reprs: Option> = groups.iter().map(|group| repr(&key(group[0]))).collect(); - let Some(reprs) = reprs else { - return KeyedGroups { - groups, - repr_ordered: false, - }; - }; - let mut ordered: Vec<(String, Vec)> = reprs.into_iter().zip(groups).collect(); + let mut ordered: Vec<(String, Vec)> = groups + .into_iter() + .map(|group| (repr(&key(group[0])), group)) + .collect(); ordered.sort_by(|left, right| left.0.cmp(&right.0)); KeyedGroups { groups: ordered.into_iter().map(|(_repr, group)| group).collect(), - repr_ordered: true, } } diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/paths.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/paths.rs index 38b631e5f0..8b3e6a778e 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/paths.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/paths.rs @@ -3,14 +3,19 @@ use crate::scope_index::constants::{ CURRENT_PATH_COMPONENT, PARENT_PATH_COMPONENT, PATH_SEPARATOR, }; -use crate::scope_index::models::ScopeDeferral; +use crate::scope_index::models::{RejectedPath, ScopeDeferral, ScopePathFailure}; /// Python's `normalize_path`; a path Python rejects defers the stage so Python raises its error. pub(crate) fn normalize_path(display_path: &str) -> Result { + normalized_path(display_path).map_err(|_| deferral(display_path)) +} + +/// Python's `normalize_path`, or why it raises `InvalidScopePathError`. +pub(crate) fn normalized_path(display_path: &str) -> Result { let raw: String = display_path.replace('\\', "/"); let has_drive: bool = raw.chars().nth(1) == Some(':'); if raw.is_empty() || raw.starts_with(PATH_SEPARATOR) || has_drive { - return Err(deferral(display_path)); + return Err(ScopePathFailure::NotProjectRelative); } let mut components: Vec<&str> = Vec::new(); for component in raw.split(PATH_SEPARATOR) { @@ -19,7 +24,7 @@ pub(crate) fn normalize_path(display_path: &str) -> Result ScopeDeferral { reason: format!("scope path Python rejects: {display_path}"), } } + +/// Python's `normalize_path`, or the path it rejects with why. +pub(crate) fn checked_path(display_path: &str) -> Result { + normalized_path(display_path).map_err(|failure| RejectedPath { + failure, + path: display_path.to_owned(), + }) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/visibility.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/visibility.rs index d9e33f22ee..da4c924a61 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/visibility.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/visibility.rs @@ -1,14 +1,16 @@ //! Python's `build_visibility_index`, `_visible_positions` and `_visibility_reason`. use crate::scope_index::_helpers::paths::{ - canonical_ancestors, is_canonical_descendant, normalize_path, parent, + canonical_ancestors, checked_path, is_canonical_descendant, normalize_path, parent, +}; +use crate::scope_index::constants::{ + CURRENT_PATH_COMPONENT, PATH_RESOURCE_PREFIX, PATH_RESOURCE_SUFFIX, }; -use crate::scope_index::constants::CURRENT_PATH_COMPONENT; -use crate::scope_index::models::ScopeKind; use crate::scope_index::models::{ - DeclarationRecord, ResourceIdentity, ResourceRecord, ScopeDeferral, ScopeIndex, - VisibilityGroups, + DeclarationRecord, InaccessibleReason, RejectedPath, ResourceIdentity, ResourceRecord, + ScopeDeferral, ScopeIndex, VisibilityGroups, VisibilityReason, }; +use crate::scope_index::models::{DeclaredScope, ResourceKind, ScopeKind}; use std::collections::HashMap; use std::hash::Hash; @@ -119,17 +121,107 @@ pub(crate) fn path_visibility( path_resource: &ResourceIdentity, folder: &str, ) -> Result { - Ok(match declaration.scope { + scope_visibility( + &DeclaredScope { + scope: declaration.scope, + owner: declaration.identity.owner.clone(), + owning_path: declaration.owning_path.clone(), + }, + path_resource, + folder, + ) +} + +/// Python's `path_visibility_reason(...) is not None` for an authored consumer path. +pub(crate) fn authored_path_visibility( + declared: &DeclaredScope, + path: &str, +) -> Result { + let normalized: String = normalize_path(path)?; + let path_resource: ResourceIdentity = ResourceIdentity { + kind: ResourceKind::Model, + name: format!("{PATH_RESOURCE_PREFIX}{normalized}{PATH_RESOURCE_SUFFIX}"), + }; + scope_visibility(declared, &path_resource, &parent(&normalized)?) +} + +fn scope_visibility( + declared: &DeclaredScope, + path_resource: &ResourceIdentity, + folder: &str, +) -> Result { + Ok(match declared.scope { ScopeKind::Global => true, - ScopeKind::Private => declaration.identity.owner.as_ref() == Some(path_resource), - ScopeKind::Local => folder == owner_path(declaration)?, - ScopeKind::Inherited => is_canonical_descendant(folder, &owner_path(declaration)?), + ScopeKind::Private => declared.owner.as_ref() == Some(path_resource), + ScopeKind::Local => folder == owner_folder(declared.owning_path.as_deref())?, + ScopeKind::Inherited => { + is_canonical_descendant(folder, &owner_folder(declared.owning_path.as_deref())?) + } }) } /// Python's `_declaration_owner`. fn owner_path(declaration: &DeclarationRecord) -> Result { + owner_folder(declaration.owning_path.as_deref()) +} + +fn owner_folder(owning_path: Option<&str>) -> Result { normalize_path( + owning_path + .filter(|path| !path.is_empty()) + .unwrap_or(CURRENT_PATH_COMPONENT), + ) +} + +/// Python's `_visibility_reason` of a declaration for a resource, with checked paths. +pub(crate) fn visibility_reason( + declaration: &DeclarationRecord, + resource: &ResourceRecord, +) -> Result, RejectedPath> { + match declaration.scope { + ScopeKind::Global => return Ok(Some(VisibilityReason::Global)), + ScopeKind::Private => { + return Ok( + (declaration.identity.owner.as_ref() == Some(&resource.identity)) + .then_some(VisibilityReason::PrivateOwner), + ); + } + ScopeKind::Local | ScopeKind::Inherited => {} + } + let owner: String = checked_owner(declaration)?; + let folder: String = checked_parent(&resource.path)?; + Ok(match declaration.scope { + ScopeKind::Local => (folder == owner).then_some(VisibilityReason::LocalOwner), + _ => { + is_canonical_descendant(&folder, &owner).then_some(VisibilityReason::InheritedAncestor) + } + }) +} + +/// Python's `_inaccessible_reason` of a declaration a resource cannot see. +pub(crate) fn inaccessible_reason( + declaration: &DeclarationRecord, + resource: &ResourceRecord, +) -> Result { + match declaration.scope { + ScopeKind::Private => return Ok(InaccessibleReason::PrivateOwner), + ScopeKind::Local => return Ok(InaccessibleReason::LocalBoundary), + ScopeKind::Global | ScopeKind::Inherited => {} + } + let owner: String = checked_owner(declaration)?; + let folder: String = checked_parent(&resource.path)?; + Ok(if is_canonical_descendant(&owner, &folder) { + InaccessibleReason::DescendantScope + } else if declaration.ownership_root.path == resource.ownership_root.path { + InaccessibleReason::SiblingScope + } else { + InaccessibleReason::UnrelatedScope + }) +} + +/// Python's `_declaration_owner`, or the owning path it rejects. +pub(crate) fn checked_owner(declaration: &DeclarationRecord) -> Result { + checked_path( declaration .owning_path .as_deref() @@ -137,3 +229,11 @@ fn owner_path(declaration: &DeclarationRecord) -> Result .unwrap_or(CURRENT_PATH_COMPONENT), ) } + +fn checked_parent(path: &str) -> Result { + let normalized: String = checked_path(path)?; + Ok(match normalized.rfind('/') { + Some(index) => normalized[..index].to_owned(), + None => CURRENT_PATH_COMPONENT.to_owned(), + }) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/constants.rs b/crates/sqlbuild-scopes/src/scope_index/constants.rs index 117a8ccc14..e8bc8ac087 100644 --- a/crates/sqlbuild-scopes/src/scope_index/constants.rs +++ b/crates/sqlbuild-scopes/src/scope_index/constants.rs @@ -28,3 +28,7 @@ pub const MACRO_TEST_LEXICAL_FILE: &str = "__macro_test__.sql"; pub const PATH_RESOURCE_PREFIX: &str = " Result { + let mut granted: HashMap<&DeclarationIdentity, Vec<&ReportGrant>> = HashMap::new(); + if let Some(group) = lookup + .groups + .grants_by_resource + .groups + .iter() + .find(|group| records.grants[group[0]].resource == resource.identity) + { + for grant in group { + let record: &ReportGrant = &records.grants[*grant]; + granted.entry(&record.declaration).or_default().push(record); + } + } + let mut classified: ReportClassification = ReportClassification::default(); + for position in &lookup.declarations { + let declaration = &records.declarations[*position]; + let reason: Option = visibility_reason(declaration, resource)?; + if let Some(reason) = reason { + classified.direct.push((*position, reason)); + } + match granted.get(&declaration.identity) { + Some(grants) if declaration.scope != ScopeKind::Private => { + let tested: bool = grants + .iter() + .all(|grant| grant.kind == GrantKind::TestedMacro); + let mut through: Vec = grants + .iter() + .map(|grant| qualified_text(&grant.through)) + .collect(); + through.sort(); + classified.relationships.push(RelationshipEntry { + position: *position, + reason: if tested { + VisibilityReason::TestedMacro + } else { + VisibilityReason::ExpectedModel + }, + through: through.join(","), + }); + } + _ if reason.is_none() => classified + .unavailable + .push((*position, inaccessible_reason(declaration, resource)?)), + _ => {} + } + } + let key = |position: usize| -> (String, String) { + let record = &records.declarations[position]; + (declaration_text(&record.identity), record.path.clone()) + }; + classified + .direct + .sort_by_cached_key(|(position, _)| key(*position)); + classified + .relationships + .sort_by_cached_key(|entry| key(entry.position)); + classified + .unavailable + .sort_by_cached_key(|(position, _)| key(*position)); + Ok(classified) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/explain_report_declaration.rs b/crates/sqlbuild-scopes/src/scope_index/main/explain_report_declaration.rs new file mode 100644 index 0000000000..3321e292b6 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/explain_report_declaration.rs @@ -0,0 +1,55 @@ +//! The scope explanation's standing of one declaration for a resource. + +use crate::scope_index::_helpers::identities::qualified_text; +use crate::scope_index::_helpers::visibility::{inaccessible_reason, visibility_reason}; +use crate::scope_index::models::{ + DeclarationRecord, DeclarationStanding, GrantKind, RejectedPath, ReportGrant, ReportLookup, + ReportRecords, ResourceRecord, VisibilityReason, +}; + +/// Visible by scope, else granted by the resource's relationships, else why it is hidden. +pub fn explain_report_declaration( + records: &ReportRecords, + lookup: &ReportLookup, + resource: &ResourceRecord, + declaration: &DeclarationRecord, +) -> Result { + if let Some(reason) = visibility_reason(declaration, resource)? { + return Ok(DeclarationStanding::Visible(reason)); + } + let mut matching: Vec<&ReportGrant> = Vec::new(); + if let Some(group) = lookup + .groups + .grants_by_resource + .groups + .iter() + .find(|group| records.grants[group[0]].resource == resource.identity) + { + for grant in group { + let record: &ReportGrant = &records.grants[*grant]; + if record.declaration == declaration.identity { + matching.push(record); + } + } + } + if matching.is_empty() { + return Ok(DeclarationStanding::Inaccessible(inaccessible_reason( + declaration, + resource, + )?)); + } + let reason: VisibilityReason = if matching + .iter() + .all(|grant| grant.kind == GrantKind::TestedMacro) + { + VisibilityReason::TestedMacro + } else { + VisibilityReason::ExpectedModel + }; + let mut through: Vec = matching + .iter() + .map(|grant| qualified_text(&grant.through)) + .collect(); + through.sort(); + Ok(DeclarationStanding::Granted(reason, through.join(","))) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/format_identity.rs b/crates/sqlbuild-scopes/src/scope_index/main/format_identity.rs new file mode 100644 index 0000000000..5395168c9b --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/format_identity.rs @@ -0,0 +1,13 @@ +//! Python's `format_identity` of a resource or declaration. + +use crate::scope_index::_helpers::identities::{declaration_text, resource_text}; +use crate::scope_index::models::QualifiedIdentity; + +/// `kind:name`, or `kind:owner_kind:owner_name.name` for a private declaration. +#[must_use] +pub fn format_identity(identity: &QualifiedIdentity) -> String { + match identity { + QualifiedIdentity::Resource(resource) => resource_text(resource), + QualifiedIdentity::Declaration(declaration) => declaration_text(declaration), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/identity_repr.rs b/crates/sqlbuild-scopes/src/scope_index/main/identity_repr.rs new file mode 100644 index 0000000000..3d06c176fc --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/identity_repr.rs @@ -0,0 +1,10 @@ +//! Python's dataclass `repr` of a resource or declaration identity, as `str()` prints it. + +use crate::scope_index::_helpers::identities::qualified_repr; +use crate::scope_index::models::QualifiedIdentity; + +/// `ResourceIdentity(kind=, name='orders')` and its declaration form. +#[must_use] +pub fn identity_repr(identity: &QualifiedIdentity) -> String { + qualified_repr(identity) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs index 7989fe667c..6e5a72df95 100644 --- a/crates/sqlbuild-scopes/src/scope_index/main/mod.rs +++ b/crates/sqlbuild-scopes/src/scope_index/main/mod.rs @@ -1,4 +1,13 @@ pub mod build_scope_index; +pub mod classify_report_resource; pub mod classify_resource; +pub mod explain_report_declaration; +pub mod format_identity; +pub mod identity_repr; +pub mod normalize_scope_path; +pub mod parse_identity; +pub mod path_visible; pub mod relationship_grants; -pub mod scope_lookup; +pub mod report_lookup; +pub mod report_target; +pub mod resolve_report_visibility; diff --git a/crates/sqlbuild-scopes/src/scope_index/main/normalize_scope_path.rs b/crates/sqlbuild-scopes/src/scope_index/main/normalize_scope_path.rs new file mode 100644 index 0000000000..e2e09b9d4e --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/normalize_scope_path.rs @@ -0,0 +1,9 @@ +//! Python's scope `normalize_path`. + +use crate::scope_index::_helpers::paths::normalized_path; +use crate::scope_index::models::ScopePathFailure; + +/// The normalized project-relative path, or why Python rejects `path`. +pub fn normalize_scope_path(path: &str) -> Result { + normalized_path(path) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/parse_identity.rs b/crates/sqlbuild-scopes/src/scope_index/main/parse_identity.rs new file mode 100644 index 0000000000..3872bdfc48 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/parse_identity.rs @@ -0,0 +1,56 @@ +//! Python's `parse_identity`: a `kind:name` or `kind:owner_kind:owner.name` identity. + +use crate::scope_index::constants::{IDENTITY_SEPARATOR, OWNER_SEPARATOR}; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationKind, IdentityFailure, QualifiedIdentity, ResourceIdentity, + ResourceKind, +}; + +/// The identity `value` names, or why Python rejects it. +pub fn parse_identity(value: &str) -> Result { + let parts: Vec<&str> = value.split(IDENTITY_SEPARATOR).collect(); + match parts.as_slice() { + [raw_kind, name] => { + if name.is_empty() { + return Err(IdentityFailure::NoName); + } + if let Some(kind) = ResourceKind::parse(raw_kind) { + return Ok(QualifiedIdentity::Resource(ResourceIdentity { + kind, + name: (*name).to_owned(), + })); + } + DeclarationKind::parse(raw_kind) + .map(|kind| { + QualifiedIdentity::Declaration(DeclarationIdentity { + kind, + name: (*name).to_owned(), + owner: None, + }) + }) + .ok_or_else(|| IdentityFailure::UnknownKind((*raw_kind).to_owned())) + } + [raw_kind, raw_owner_kind, owner_and_name] => { + let (Some(kind), Some(owner_kind)) = ( + DeclarationKind::parse(raw_kind), + ResourceKind::parse(raw_owner_kind), + ) else { + return Err(IdentityFailure::InvalidPrivate); + }; + match owner_and_name.rsplit_once(OWNER_SEPARATOR) { + Some((owner_name, name)) if !owner_name.is_empty() && !name.is_empty() => { + Ok(QualifiedIdentity::Declaration(DeclarationIdentity { + kind, + name: name.to_owned(), + owner: Some(ResourceIdentity { + kind: owner_kind, + name: owner_name.to_owned(), + }), + })) + } + _ => Err(IdentityFailure::InvalidPrivate), + } + } + _ => Err(IdentityFailure::Unqualified), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/path_visible.rs b/crates/sqlbuild-scopes/src/scope_index/main/path_visible.rs new file mode 100644 index 0000000000..9fdcaff525 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/path_visible.rs @@ -0,0 +1,9 @@ +//! Python's `declaration_visibility(...) is not None` for an authored consumer path. + +use crate::scope_index::_helpers::visibility::authored_path_visibility; +use crate::scope_index::models::{DeclaredScope, ScopeDeferral}; + +/// Whether a declaration with these scope facts is visible from the authored `path`. +pub fn path_visible(declared: &DeclaredScope, path: &str) -> Result { + authored_path_visibility(declared, path) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/report_lookup.rs b/crates/sqlbuild-scopes/src/scope_index/main/report_lookup.rs new file mode 100644 index 0000000000..388c9375f0 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/report_lookup.rs @@ -0,0 +1,151 @@ +//! Python's `build_lookup` over a full scope index's records. + +use crate::scope_index::_helpers::identities::{ + declaration_repr, declaration_text, qualified_repr, qualified_text, resource_repr, + resource_text, +}; +use crate::scope_index::_helpers::ordering::keyed_groups; +use crate::scope_index::_helpers::paths::checked_path; +use crate::scope_index::_helpers::visibility::{VisibilityIndex, checked_owner}; +use crate::scope_index::models::{ + RejectedPath, ReportGroups, ReportLookup, ReportRecords, ScopeIndex, ScopeKind, +}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +/// Stable `sorted(range(count), key=key)`. +fn sorted_positions(count: usize, key: impl Fn(usize) -> K) -> Vec { + let mut positions: Vec = (0..count).collect(); + positions.sort_by_cached_key(|position| key(*position)); + positions +} + +/// Every canonical order, lookup mapping and visibility group, or the path Python rejects first. +pub fn report_lookup(records: &ReportRecords) -> Result { + let resource_paths: Vec = records + .resources + .iter() + .map(|record| checked_path(&record.path)) + .collect::>()?; + let declaration_paths: Vec = records + .declarations + .iter() + .map(|record| checked_path(&record.path)) + .collect::>()?; + let resources: Vec = sorted_positions(records.resources.len(), |resource| { + ( + resource_text(&records.resources[resource].identity), + resource_paths[resource].clone(), + ) + }); + let declarations: Vec = sorted_positions(records.declarations.len(), |declaration| { + let record = &records.declarations[declaration]; + ( + declaration_text(&record.identity), + declaration_paths[declaration].clone(), + record.line.to_string(), + record.column.to_string(), + ) + }); + for declaration in &declarations { + let record = &records.declarations[*declaration]; + if matches!(record.scope, ScopeKind::Local | ScopeKind::Inherited) { + checked_owner(record)?; + } + } + let usages: Vec = sorted_positions(records.usages.len(), |usage| { + let record = &records.usages[usage]; + ( + qualified_text(&record.consumer), + declaration_text(&record.declaration), + record.kind.clone(), + record + .through + .as_ref() + .map(qualified_text) + .unwrap_or_default(), + ) + }); + let grants: Vec = sorted_positions(records.grants.len(), |grant| { + let record = &records.grants[grant]; + ( + resource_text(&record.resource), + declaration_text(&record.declaration), + qualified_text(&record.through), + record.kind.as_str(), + ) + }); + let roots: Vec = sorted_positions(records.roots.len(), |root| { + let record = &records.roots[root]; + ( + record.path.clone(), + record + .resource_kind + .map(|kind| kind.as_str()) + .unwrap_or_default(), + ) + }); + let diagnostics: Vec = sorted_positions(records.diagnostics.len(), |diagnostic| { + let record = &records.diagnostics[diagnostic]; + ( + record.path.clone().unwrap_or_default(), + record.line.unwrap_or(0), + record.column.unwrap_or(0), + record.code.clone(), + ) + }); + let groups: ReportGroups = ReportGroups { + resources: keyed_groups( + &resources, + |resource| &records.resources[resource].identity, + |identity| resource_repr(identity), + ), + resources_by_path: keyed_groups( + &resources, + |resource| resource_paths[resource].as_str(), + |path| python_str_repr(active_python_text(), path), + ), + declarations: keyed_groups( + &declarations, + |declaration| &records.declarations[declaration].identity, + |identity| declaration_repr(identity), + ), + usages_by_consumer: keyed_groups( + &usages, + |usage| &records.usages[usage].consumer, + |identity| qualified_repr(identity), + ), + usages_by_declaration: keyed_groups( + &usages, + |usage| &records.usages[usage].declaration, + |identity| declaration_repr(identity), + ), + grants_by_resource: keyed_groups( + &grants, + |grant| &records.grants[grant].resource, + |identity| resource_repr(identity), + ), + }; + let index: ScopeIndex = ScopeIndex { + declarations: records.declarations.clone(), + canonical_declarations: declarations.clone(), + ..ScopeIndex::default() + }; + let visibility = VisibilityIndex::build(&index) + .map_err(|_| RejectedPath { + failure: crate::scope_index::models::ScopePathFailure::NotProjectRelative, + path: String::new(), + })? + .groups(); + Ok(ReportLookup { + roots, + resources, + declarations, + usages, + grants, + diagnostics, + groups, + visibility, + resource_paths, + }) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/report_target.rs b/crates/sqlbuild-scopes/src/scope_index/main/report_target.rs new file mode 100644 index 0000000000..85e98c6ec5 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/report_target.rs @@ -0,0 +1,51 @@ +//! Python's `query_target`: the resources or declarations a scope query names. + +use crate::scope_index::_helpers::paths::checked_path; +use crate::scope_index::main::parse_identity::parse_identity; +use crate::scope_index::models::{ + KeyedGroups, QualifiedIdentity, ReportLookup, ReportRecords, ReportTarget, TargetFailure, + TargetMatch, +}; + +/// The group of `groups` whose first member `is_key`, else no members. +fn group_of(groups: &KeyedGroups, is_key: impl Fn(usize) -> bool) -> Vec { + groups + .groups + .iter() + .find(|group| is_key(group[0])) + .cloned() + .unwrap_or_default() +} + +/// The matching records' positions in lookup order, or why Python raises. +pub fn report_target( + records: &ReportRecords, + lookup: &ReportLookup, + target: &ReportTarget, +) -> Result { + let identity: QualifiedIdentity = match target { + ReportTarget::Identity(identity) => identity.clone(), + ReportTarget::Text(text) if text.contains(':') => { + parse_identity(text).map_err(TargetFailure::Identity)? + } + ReportTarget::Text(text) => { + let path: String = checked_path(text).map_err(TargetFailure::Path)?; + return Ok(TargetMatch::Resources(group_of( + &lookup.groups.resources_by_path, + |resource| matches!(checked_path(&records.resources[resource].path), Ok(found) if found == path), + ))); + } + }; + Ok(match identity { + QualifiedIdentity::Resource(identity) => { + TargetMatch::Resources(group_of(&lookup.groups.resources, |resource| { + records.resources[resource].identity == identity + })) + } + QualifiedIdentity::Declaration(identity) => { + TargetMatch::Declarations(group_of(&lookup.groups.declarations, |declaration| { + records.declarations[declaration].identity == identity + })) + } + }) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/resolve_report_visibility.rs b/crates/sqlbuild-scopes/src/scope_index/main/resolve_report_visibility.rs new file mode 100644 index 0000000000..5b662db98d --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/main/resolve_report_visibility.rs @@ -0,0 +1,63 @@ +//! Python's `_classify_resource` and `_inaccessible_reason` over a full scope lookup. + +use std::collections::HashMap; + +use crate::scope_index::_helpers::visibility::{inaccessible_reason, visibility_reason}; +use crate::scope_index::models::{ + DeclarationIdentity, GrantKind, RejectedPath, ReportLookup, ReportRecords, ResolvedVisibility, + ResolvedVisible, ResourceRecord, VisibilityReason, +}; + +/// Every canonical declaration visible to `resource`, by scope or grant, and the rest. +pub fn resolve_report_visibility( + records: &ReportRecords, + lookup: &ReportLookup, + resource: &ResourceRecord, +) -> Result { + let mut grants: HashMap<&DeclarationIdentity, Vec> = HashMap::new(); + if let Some(group) = lookup + .groups + .grants_by_resource + .groups + .iter() + .find(|group| records.grants[group[0]].resource == resource.identity) + { + for grant in group { + grants + .entry(&records.grants[*grant].declaration) + .or_default() + .push(*grant); + } + } + let mut resolved: ResolvedVisibility = ResolvedVisibility::default(); + for (position, declaration) in lookup.declarations.iter().enumerate() { + let record = &records.declarations[*declaration]; + let reason: Option = visibility_reason(record, resource)?; + if let Some(reason) = reason { + resolved.visible.push(ResolvedVisible { + position, + reason, + grant: None, + }); + } + match grants.get(&record.identity) { + Some(granted) => { + for grant in granted { + resolved.visible.push(ResolvedVisible { + position, + reason: match records.grants[*grant].kind { + GrantKind::TestedMacro => VisibilityReason::TestedMacro, + GrantKind::ExpectedModel => VisibilityReason::ExpectedModel, + }, + grant: Some(*grant), + }); + } + } + None if reason.is_none() => resolved + .inaccessible + .push((position, inaccessible_reason(record, resource)?)), + None => {} + } + } + Ok(resolved) +} diff --git a/crates/sqlbuild-scopes/src/scope_index/main/scope_lookup.rs b/crates/sqlbuild-scopes/src/scope_index/main/scope_lookup.rs deleted file mode 100644 index 616304228a..0000000000 --- a/crates/sqlbuild-scopes/src/scope_index/main/scope_lookup.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! The groupings Python's `build_lookup` derives from an index and its relationship grants. - -use crate::scope_index::_helpers::identities::{ - declaration_repr, declaration_text, grant_through_text, resource_repr, resource_text, -}; -use crate::scope_index::_helpers::ordering::{keyed_groups, sorted_by_key}; -use crate::scope_index::_helpers::paths::normalize_path; -use crate::scope_index::_helpers::visibility::VisibilityIndex; -use crate::scope_index::models::{ - DeclarationIdentity, GrantRecord, ScopeDeferral, ScopeIndex, ScopeLookupGroups, -}; -use sqlbuild_core::text::main::python_str_repr::python_str_repr; - -/// Every lookup grouping of `index` once `grants` are attached to it. -pub fn scope_lookup_groups( - index: &ScopeIndex, - grants: &[GrantRecord], -) -> Result { - let dependency = |usage: usize| -> &DeclarationIdentity { - let record = index.usages[usage]; - &index.declarations[record.consumer].dependencies[record.dependency] - }; - let consumer = |usage: usize| -> &DeclarationIdentity { - &index.declarations[index.usages[usage].consumer].identity - }; - let usage_indexes: Vec = (0..index.usages.len()).collect(); - let canonical_usages: Vec = sorted_by_key(&usage_indexes, |usage| { - Ok(( - declaration_text(consumer(usage)), - declaration_text(dependency(usage)), - )) - })?; - let grant_indexes: Vec = (0..grants.len()).collect(); - let canonical_grants: Vec = sorted_by_key(&grant_indexes, |grant| { - let record: &GrantRecord = &grants[grant]; - Ok(( - resource_text(&record.resource), - declaration_text(&index.declarations[record.declaration].identity), - grant_through_text(index, record), - record.kind.as_str(), - )) - })?; - let path_keys: Vec = index - .resources - .iter() - .map(|record| normalize_path(&record.path)) - .collect::>()?; - Ok(ScopeLookupGroups { - resources: keyed_groups( - &index.canonical_resources, - |resource| &index.resources[resource].identity, - |identity| resource_repr(identity), - ), - resources_by_path: keyed_groups( - &index.canonical_resources, - |resource| path_keys[resource].as_str(), - |path| python_str_repr(path), - ), - declarations: keyed_groups( - &index.canonical_declarations, - |declaration| &index.declarations[declaration].identity, - |identity| declaration_repr(identity), - ), - usages_by_consumer: keyed_groups(&canonical_usages, consumer, |identity| { - declaration_repr(identity) - }), - usages_by_declaration: keyed_groups(&canonical_usages, dependency, |identity| { - declaration_repr(identity) - }), - grants_by_resource: keyed_groups( - &canonical_grants, - |grant| &grants[grant].resource, - |identity| resource_repr(identity), - ), - visibility: VisibilityIndex::build(index)?.groups(), - canonical_usages, - canonical_grants, - }) -} diff --git a/crates/sqlbuild-scopes/src/scope_index/models.rs b/crates/sqlbuild-scopes/src/scope_index/models.rs index cf0a49dde0..b5f80ce53d 100644 --- a/crates/sqlbuild-scopes/src/scope_index/models.rs +++ b/crates/sqlbuild-scopes/src/scope_index/models.rs @@ -95,6 +95,20 @@ impl DeclarationKind { } } + /// Python's `DECLARATION_ROLE_PARTS`: the folders a declaration of this kind lives under. + pub fn role_parts(self) -> &'static [&'static str] { + match self { + Self::Macro => &["macros"], + Self::Enum => &["enums"], + Self::Constant => &["constants"], + Self::Audit => &["audits", "generic"], + Self::SingularAudit => &["audits", "singular"], + Self::Schema => &["schemas"], + Self::SqlHook => &["hooks", "sql"], + Self::PythonHook => &["hooks", "python"], + } + } + /// The kind with Python's enum value `value`. pub fn parse(value: &str) -> Option { [ @@ -155,6 +169,13 @@ impl OwnershipRootKind { Self::Global => "global", } } + + /// The kind with Python's enum `value`. + pub fn parse(value: &str) -> Option { + [Self::Resource, Self::Global] + .into_iter() + .find(|kind| kind.as_str() == value) + } } /// Python's `GrantKind`. @@ -354,9 +375,8 @@ pub struct GrantRecord { /// One lookup mapping's groups; members index the records the mapping groups. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct KeyedGroups { + /// Groups in Python's `repr(key)` order. pub groups: Vec>, - /// Whether `groups` follow Python's `repr(key)` order; otherwise first-occurrence order. - pub repr_ordered: bool, } /// Python's `DeclarationVisibilityIndex` positions over the canonical declarations. @@ -369,20 +389,6 @@ pub struct VisibilityGroups { pub inherited: Vec<(String, Vec)>, } -/// Every grouping Python's `build_lookup` derives from one index and its grants. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct ScopeLookupGroups { - pub canonical_usages: Vec, - pub canonical_grants: Vec, - pub resources: KeyedGroups, - pub resources_by_path: KeyedGroups, - pub declarations: KeyedGroups, - pub usages_by_consumer: KeyedGroups, - pub usages_by_declaration: KeyedGroups, - pub grants_by_resource: KeyedGroups, - pub visibility: VisibilityGroups, -} - /// The native scope stage cannot reproduce Python for this project; the Python stage runs instead. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ScopeDeferral { @@ -455,3 +461,197 @@ pub struct ResourceVisibility { pub visible: Vec, pub inaccessible: Vec, } + +/// The scope facts that decide whether a declaration is visible from an authored path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclaredScope { + pub scope: ScopeKind, + /// The owning resource of a private declaration. + pub owner: Option, + pub owning_path: Option, +} + +/// A qualified identity as Python's `parse_identity` reads it. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum QualifiedIdentity { + Resource(ResourceIdentity), + Declaration(DeclarationIdentity), +} + +/// Why Python's `parse_identity` rejects a value; each names the value or the raw kind. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum IdentityFailure { + /// `Qualified identity has no name: {value!r}`. + NoName, + /// `Unknown qualified identity kind: {raw_kind!r}`. + UnknownKind(String), + /// `Invalid private identity: {value!r}`. + InvalidPrivate, + /// `Expected a kind-qualified identity such as 'model:name': {value!r}`. + Unqualified, +} + +/// Why Python's `normalize_path` raises `InvalidScopePathError`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScopePathFailure { + /// `Scope path must be project-relative: {path}`. + NotProjectRelative, + /// `Scope path escapes the project: {path}`. + EscapesProject, +} + +/// One usage of a full scope index: consumer, declaration, usage kind and what it came through. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReportUsage { + pub consumer: QualifiedIdentity, + pub declaration: DeclarationIdentity, + pub kind: String, + pub through: Option, +} + +/// One relationship grant of a full scope index. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReportGrant { + pub resource: ResourceIdentity, + pub declaration: DeclarationIdentity, + pub through: QualifiedIdentity, + pub kind: GrantKind, +} + +/// The parts of a diagnostic the lookup orders diagnostics by. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReportDiagnosticKey { + pub path: Option, + pub line: Option, + pub column: Option, + pub code: String, +} + +/// A full scope index's records as Python's `ScopeIndex` holds them, in its tuple orders. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ReportRecords { + pub roots: Vec, + pub resources: Vec, + pub declarations: Vec, + pub usages: Vec, + pub grants: Vec, + pub diagnostics: Vec, +} + +/// Python's `build_lookup` over report records: canonical orders, groups and visibility. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ReportLookup { + pub roots: Vec, + pub resources: Vec, + pub declarations: Vec, + pub usages: Vec, + pub grants: Vec, + pub diagnostics: Vec, + pub groups: ReportGroups, + pub visibility: VisibilityGroups, + /// Each input resource's normalized path, the `resources_by_path` key. + pub resource_paths: Vec, +} + +/// The lookup's mappings as groups of record positions. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ReportGroups { + pub resources: KeyedGroups, + pub resources_by_path: KeyedGroups, + pub declarations: KeyedGroups, + pub usages_by_consumer: KeyedGroups, + pub usages_by_declaration: KeyedGroups, + pub grants_by_resource: KeyedGroups, +} + +/// A path Python's `normalize_path` rejects, and why. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RejectedPath { + pub failure: ScopePathFailure, + pub path: String, +} + +/// What a scope query targets: an identity, or text naming one or a resource path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ReportTarget { + Identity(QualifiedIdentity), + Text(String), +} + +/// Python's `ScopeTargetQuery` matches as record positions in lookup group order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TargetMatch { + Resources(Vec), + Declarations(Vec), +} + +/// Why Python's `query_target` raises. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TargetFailure { + Identity(IdentityFailure), + Path(RejectedPath), +} + +/// Python's `InaccessibleReason`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum InaccessibleReason { + LocalBoundary, + SiblingScope, + DescendantScope, + UnrelatedScope, + PrivateOwner, +} + +impl InaccessibleReason { + /// Python's enum value text. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::LocalBoundary => "local_boundary", + Self::SiblingScope => "sibling_scope", + Self::DescendantScope => "descendant_scope", + Self::UnrelatedScope => "unrelated_scope", + Self::PrivateOwner => "private_owner", + } + } +} + +/// One relationship-visible declaration: its position, reason and sorted `through` text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RelationshipEntry { + pub position: usize, + pub reason: VisibilityReason, + pub through: String, +} + +/// The report's `_classify`: direct, relationship and unavailable declarations, each sorted. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ReportClassification { + pub direct: Vec<(usize, VisibilityReason)>, + pub relationships: Vec, + pub unavailable: Vec<(usize, InaccessibleReason)>, +} + +/// One visible fact: a canonical declaration position, the reason, and the granting grant. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ResolvedVisible { + pub position: usize, + pub reason: VisibilityReason, + pub grant: Option, +} + +/// Python's `_classify_resource` with each inaccessible position's reason. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResolvedVisibility { + pub visible: Vec, + pub inaccessible: Vec<(usize, InaccessibleReason)>, +} + +/// How one declaration stands for a resource in a scope explanation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DeclarationStanding { + Visible(VisibilityReason), + /// Granted by relationships: the reason and the sorted `through` identities joined by commas. + Granted(VisibilityReason, String), + Inaccessible(InaccessibleReason), +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests.rs b/crates/sqlbuild-scopes/src/scope_index/tests.rs index eff62179a5..8a8e696f2d 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests.rs @@ -4,9 +4,13 @@ mod build_scope_index; mod classify_resource; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_parse_identity.rs"] +mod parse_identity; +#[path = "tests/test_path_visible.rs"] +mod path_visible; #[path = "tests/test_relationship_grants.rs"] mod relationship_grants; -#[path = "tests/test_scope_lookup.rs"] -mod scope_lookup; +#[path = "tests/test_report_lookup.rs"] +mod report_lookup; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs b/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs index 7d85257420..bebc208cef 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests/helpers.rs @@ -6,11 +6,11 @@ use std::collections::HashMap; use crate::scope_index::main::build_scope_index::build_scope_index; use crate::scope_index::main::classify_resource::classify_resource; use crate::scope_index::main::relationship_grants::relationship_grants; -use crate::scope_index::main::scope_lookup::scope_lookup_groups; +use crate::scope_index::main::report_lookup::report_lookup; use crate::scope_index::models::{ ConsumerResource, DeclarationIdentity, DeclarationInput, DeclarationKind, GrantEntry, - RelationshipFact, ResourceIdentity, ResourceInput, ResourceKind, ResourceRoot, - ResourceVisibility, ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, ScopeLookupGroups, + RelationshipFact, ReportLookup, ReportRecords, ResourceIdentity, ResourceInput, ResourceKind, + ResourceRoot, ResourceVisibility, ScopeDeferral, ScopeIndex, ScopeInputs, ScopeKind, VisibilityTable, }; use crate::scope_index::tests::test_types::{ @@ -130,26 +130,33 @@ pub(super) fn grant_texts(test_case: &GrantTestCase) -> Vec { .collect() } -/// The first resource identity of each resource group of a lookup case, and its repr flag. -pub(super) fn resource_groups(test_case: &LookupTestCase) -> (Vec, bool) { +/// The first resource identity of each resource group of a lookup case. +pub(super) fn resource_groups(test_case: &LookupTestCase) -> Vec { let index: ScopeIndex = built_index(test_case.resources, test_case.declarations); - let groups: ScopeLookupGroups = scope_lookup_groups(&index, &[]).expect("no deferral"); - ( - groups - .resources - .groups - .iter() - .map(|group| resource_text(&index.resources[group[0]].identity)) - .collect(), - groups.resources.repr_ordered, - ) + let lookup: ReportLookup = report_lookup(&report_records(&index)).expect("valid paths"); + lookup + .groups + .resources + .groups + .iter() + .map(|group| resource_text(&index.resources[group[0]].identity)) + .collect() +} + +/// A built index's records as a full scope index's records. +fn report_records(index: &ScopeIndex) -> ReportRecords { + ReportRecords { + resources: index.resources.clone(), + declarations: index.declarations.clone(), + ..ReportRecords::default() + } } /// The visibility position groups of a lookup case as `kind owner positions` text. pub(super) fn visibility_texts(test_case: &LookupTestCase) -> Vec { let index: ScopeIndex = built_index(test_case.resources, test_case.declarations); - let visibility = scope_lookup_groups(&index, &[]) - .expect("no deferral") + let visibility = report_lookup(&report_records(&index)) + .expect("valid paths") .visibility; let mut texts: Vec = vec![format!("global {:?}", visibility.global)]; texts.extend( diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_parse_identity.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_parse_identity.rs new file mode 100644 index 0000000000..896b2ce495 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_parse_identity.rs @@ -0,0 +1,70 @@ +use crate::scope_index::main::parse_identity::parse_identity; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationKind, IdentityFailure, QualifiedIdentity, ResourceIdentity, + ResourceKind, +}; +use crate::scope_index::tests::test_types::ParseIdentityTestCase; + +#[test] +fn given_qualified_value_when_parsing_then_python_identity_or_failure_is_returned() { + let test_cases = [ + ParseIdentityTestCase { + description: "resource kinds win over declaration kinds", + value: "model:orders", + expected_identity: Ok(QualifiedIdentity::Resource(ResourceIdentity { + kind: ResourceKind::Model, + name: "orders".to_owned(), + })), + }, + ParseIdentityTestCase { + description: "a public declaration", + value: "enum:order_status", + expected_identity: Ok(QualifiedIdentity::Declaration(DeclarationIdentity { + kind: DeclarationKind::Enum, + name: "order_status".to_owned(), + owner: None, + })), + }, + ParseIdentityTestCase { + description: "a private declaration splits its owner at the last dot", + value: "constant:model:orders.v2.limit", + expected_identity: Ok(QualifiedIdentity::Declaration(DeclarationIdentity { + kind: DeclarationKind::Constant, + name: "limit".to_owned(), + owner: Some(ResourceIdentity { + kind: ResourceKind::Model, + name: "orders.v2".to_owned(), + }), + })), + }, + ParseIdentityTestCase { + description: "a path folder is an unknown kind", + value: "models/a:b/orders.sql", + expected_identity: Err(IdentityFailure::UnknownKind("models/a".to_owned())), + }, + ParseIdentityTestCase { + description: "an empty name", + value: "model:", + expected_identity: Err(IdentityFailure::NoName), + }, + ParseIdentityTestCase { + description: "a private identity without an owner name", + value: "enum:model:.status", + expected_identity: Err(IdentityFailure::InvalidPrivate), + }, + ParseIdentityTestCase { + description: "too many parts", + value: "a:b:c:d", + expected_identity: Err(IdentityFailure::Unqualified), + }, + ]; + + for test_case in test_cases { + assert_eq!( + parse_identity(test_case.value), + test_case.expected_identity, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_path_visible.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_path_visible.rs new file mode 100644 index 0000000000..ff4a487007 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_path_visible.rs @@ -0,0 +1,51 @@ +use crate::scope_index::main::path_visible::path_visible; +use crate::scope_index::models::{DeclaredScope, ScopeKind}; +use crate::scope_index::tests::test_types::PathVisibleTestCase; + +#[test] +fn given_declared_scope_when_checking_authored_path_then_python_visibility_is_returned() { + let test_cases = [ + PathVisibleTestCase { + description: "global is visible everywhere", + scope: ScopeKind::Global, + owning_path: None, + path: "models/marts/orders.sql", + expected_visible: true, + }, + PathVisibleTestCase { + description: "inherited reaches a descendant folder", + scope: ScopeKind::Inherited, + owning_path: Some("models"), + path: "models/marts/orders.sql", + expected_visible: true, + }, + PathVisibleTestCase { + description: "inherited does not reach a sibling tree", + scope: ScopeKind::Inherited, + owning_path: Some("models/staging"), + path: "models/marts/orders.sql", + expected_visible: false, + }, + PathVisibleTestCase { + description: "local only reaches its own folder", + scope: ScopeKind::Local, + owning_path: Some("models"), + path: "models/marts/orders.sql", + expected_visible: false, + }, + ]; + + for test_case in test_cases { + let declared = DeclaredScope { + scope: test_case.scope, + owner: None, + owning_path: test_case.owning_path.map(str::to_owned), + }; + assert_eq!( + path_visible(&declared, test_case.path), + Ok(test_case.expected_visible), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_scope_lookup.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_report_lookup.rs similarity index 55% rename from crates/sqlbuild-scopes/src/scope_index/tests/test_scope_lookup.rs rename to crates/sqlbuild-scopes/src/scope_index/tests/test_report_lookup.rs index b89ccd9df8..8058914bad 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests/test_scope_lookup.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_report_lookup.rs @@ -1,6 +1,5 @@ use crate::scope_index::tests::helpers::{resource_groups, visibility_texts}; -use crate::scope_index::tests::test_types::{LookupTestCase, ReprTestCase}; -use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use crate::scope_index::tests::test_types::LookupTestCase; #[test] fn given_identity_keys_when_grouping_then_groups_follow_python_repr_order() { @@ -22,15 +21,29 @@ fn given_identity_keys_when_grouping_then_groups_follow_python_repr_order() { "model:ab", "seed:a", ], - expected_repr_ordered: true, expected_visibility: &["global []"], }, LookupTestCase { - description: "non-ASCII keys are left for Python to order", - resources: &[("model", "caf\u{e9}", "models/cafe.sql")], + description: "non-ASCII keys order by repr, with escapes for non-printable text", + resources: &[ + ("model", "caf\u{e9}", "models/cafe_accent.sql"), + ("model", "cafe", "models/cafe.sql"), + ("model", "\u{a0}x", "models/nbsp.sql"), + ("model", "\u{1F600}", "models/emoji.sql"), + ("model", "Zed", "models/zed.sql"), + ("model", "\u{200b}z", "models/zwsp.sql"), + ("model", "e\u{301}", "models/combining.sql"), + ], declarations: &[], - expected_resource_groups: &["model:caf\u{e9}"], - expected_repr_ordered: false, + expected_resource_groups: &[ + "model:Zed", + "model:\u{200b}z", + "model:\u{a0}x", + "model:cafe", + "model:caf\u{e9}", + "model:e\u{301}", + "model:\u{1F600}", + ], expected_visibility: &["global []"], }, LookupTestCase { @@ -64,7 +77,6 @@ fn given_identity_keys_when_grouping_then_groups_follow_python_repr_order() { ), ], expected_resource_groups: &["model:orders"], - expected_repr_ordered: true, expected_visibility: &[ "global [2]", "private model:orders [0]", @@ -77,14 +89,11 @@ fn given_identity_keys_when_grouping_then_groups_follow_python_repr_order() { for test_case in test_cases { assert_eq!( resource_groups(&test_case), - ( - test_case - .expected_resource_groups - .iter() - .map(|name| (*name).to_owned()) - .collect(), - test_case.expected_repr_ordered - ), + test_case + .expected_resource_groups + .iter() + .map(|name| (*name).to_owned()) + .collect::>(), "{}", test_case.description ); @@ -96,48 +105,3 @@ fn given_identity_keys_when_grouping_then_groups_follow_python_repr_order() { ); } } - -#[test] -fn given_key_text_when_taking_repr_then_it_matches_python() { - let test_cases = [ - ReprTestCase { - description: "plain text uses single quotes", - value: "plain", - expected_repr: Some("'plain'"), - }, - ReprTestCase { - description: "a single quote alone switches to double quotes", - value: "it's", - expected_repr: Some("\"it's\""), - }, - ReprTestCase { - description: "both quotes keep single quotes and escape them", - value: "both'\"", - expected_repr: Some("'both\\'\"'"), - }, - ReprTestCase { - description: "whitespace controls and backslashes use short escapes", - value: "tab\tnew\nline\r\\", - expected_repr: Some("'tab\\tnew\\nline\\r\\\\'"), - }, - ReprTestCase { - description: "other controls use hex escapes", - value: "bell\u{7}del\u{7f}", - expected_repr: Some("'bell\\x07del\\x7f'"), - }, - ReprTestCase { - description: "non-ASCII text is left to Python", - value: "caf\u{e9}", - expected_repr: None, - }, - ]; - - for test_case in test_cases { - assert_eq!( - python_str_repr(test_case.value).as_deref(), - test_case.expected_repr, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs index 38fa9c8b75..3d7ff5f2ab 100644 --- a/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs +++ b/crates/sqlbuild-scopes/src/scope_index/tests/test_types.rs @@ -74,18 +74,10 @@ pub(super) struct LookupTestCase { pub(super) declarations: &'static [DeclarationRow], /// First resource identity of each resource group, in mapping order. pub(super) expected_resource_groups: &'static [&'static str], - pub(super) expected_repr_ordered: bool, /// `global`, `private`, `local` and `inherited` position groups as text. pub(super) expected_visibility: &'static [&'static str], } -/// One Python `repr` case for a lookup key string. -pub(super) struct ReprTestCase { - pub(super) description: &'static str, - pub(super) value: &'static str, - pub(super) expected_repr: Option<&'static str>, -} - /// `(position, reason, through)` of one visible record. pub(super) type VisibleRow = (usize, &'static str, Option); @@ -97,3 +89,22 @@ pub(super) struct ClassifyTestCase { pub(super) grants: &'static [(u32, VisibilityReason, usize)], pub(super) expected_classified: Option<(&'static [VisibleRow], &'static [usize])>, } + +/// One declaration's scope facts, an authored consumer path, and whether it is visible. +pub(super) struct PathVisibleTestCase { + pub(super) description: &'static str, + pub(super) scope: crate::scope_index::models::ScopeKind, + pub(super) owning_path: Option<&'static str>, + pub(super) path: &'static str, + pub(super) expected_visible: bool, +} + +/// One qualified value and the identity or failure Python's `parse_identity` gives it. +pub(super) struct ParseIdentityTestCase { + pub(super) description: &'static str, + pub(super) value: &'static str, + pub(super) expected_identity: Result< + crate::scope_index::models::QualifiedIdentity, + crate::scope_index::models::IdentityFailure, + >, +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/context.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/context.rs new file mode 100644 index 0000000000..f14d24848e --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/context.rs @@ -0,0 +1,200 @@ +//! One query's view of a lookup: records, values and every mapping Python's `ScopeLookup` keys. + +use std::cell::OnceCell; +use std::collections::HashMap; + +use crate::placement::main::required_declaration_placements::required_declaration_placements; +use crate::placement::models::{PlacementIndex, PlacementUsage, RequiredPlacement}; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, KeyedGroups, OwnershipRoot, QualifiedIdentity, + ReportGrant, ReportUsage, ResourceIdentity, ResourceRecord, +}; +use crate::scope_report::models::{Completeness, DeclarationValues, ReportSource}; + +/// Each group's members by the key of its first member. +fn keyed( + groups: &KeyedGroups, + key: impl Fn(usize) -> K, +) -> HashMap { + groups + .groups + .iter() + .map(|group| (key(group[0]), group.as_slice())) + .collect() +} + +/// The lookup mappings of one query, keyed as Python's dictionaries are. +pub(crate) struct Context<'a> { + pub(crate) source: ReportSource<'a>, + resources_by_path: HashMap<&'a str, &'a [usize]>, + declarations: HashMap<&'a DeclarationIdentity, &'a [usize]>, + usages_by_consumer: HashMap<&'a QualifiedIdentity, &'a [usize]>, + usages_by_declaration: HashMap<&'a DeclarationIdentity, &'a [usize]>, + grants_by_resource: HashMap<&'a ResourceIdentity, &'a [usize]>, + /// Every grant of each declaration, in the index's canonical grant order. + grants_by_declaration: HashMap<&'a DeclarationIdentity, Vec>, + placements: OnceCell>, +} + +impl<'a> Context<'a> { + pub(crate) fn new(source: ReportSource<'a>) -> Self { + let records = source.records; + let groups = &source.lookup.groups; + let mut grants_by_declaration: HashMap<&'a DeclarationIdentity, Vec> = + HashMap::new(); + for grant in &source.lookup.grants { + grants_by_declaration + .entry(&records.grants[*grant].declaration) + .or_default() + .push(*grant); + } + Self { + source, + resources_by_path: keyed(&groups.resources_by_path, |member| { + source.lookup.resource_paths[member].as_str() + }), + declarations: keyed(&groups.declarations, |member| { + &records.declarations[member].identity + }), + usages_by_consumer: keyed(&groups.usages_by_consumer, |member| { + &records.usages[member].consumer + }), + usages_by_declaration: keyed(&groups.usages_by_declaration, |member| { + &records.usages[member].declaration + }), + grants_by_resource: keyed(&groups.grants_by_resource, |member| { + &records.grants[member].resource + }), + grants_by_declaration, + placements: OnceCell::new(), + } + } + + pub(crate) fn completeness(&self) -> Completeness { + self.source.values.completeness + } + + pub(crate) fn declaration(&self, position: usize) -> &'a DeclarationRecord { + &self.source.records.declarations[position] + } + + pub(crate) fn resource(&self, position: usize) -> &'a ResourceRecord { + &self.source.records.resources[position] + } + + pub(crate) fn values(&self, position: usize) -> Option<&'a DeclarationValues> { + self.source.values.declarations.get(position) + } + + pub(crate) fn usage(&self, position: usize) -> &'a ReportUsage { + &self.source.records.usages[position] + } + + pub(crate) fn grant(&self, position: usize) -> &'a ReportGrant { + &self.source.records.grants[position] + } + + /// The index's declarations in canonical order, as input positions. + pub(crate) fn canonical_declarations(&self) -> &'a [usize] { + &self.source.lookup.declarations + } + + /// The index's ownership roots in canonical order. + pub(crate) fn canonical_roots(&self) -> impl Iterator + 'a { + let records = self.source.records; + self.source + .lookup + .roots + .iter() + .map(move |position| &records.roots[*position]) + } + + /// The index's resources in canonical order. + pub(crate) fn canonical_resources(&self) -> impl Iterator + 'a { + let records = self.source.records; + self.source + .lookup + .resources + .iter() + .map(move |position| &records.resources[*position]) + } + + pub(crate) fn declarations_of(&self, identity: &DeclarationIdentity) -> &'a [usize] { + self.declarations.get(identity).copied().unwrap_or_default() + } + + pub(crate) fn resources_at(&self, path: &str) -> &'a [usize] { + self.resources_by_path + .get(path) + .copied() + .unwrap_or_default() + } + + pub(crate) fn usages_of_consumer(&self, consumer: &QualifiedIdentity) -> &'a [usize] { + self.usages_by_consumer + .get(consumer) + .copied() + .unwrap_or_default() + } + + pub(crate) fn usages_of_declaration(&self, declaration: &DeclarationIdentity) -> &'a [usize] { + self.usages_by_declaration + .get(declaration) + .copied() + .unwrap_or_default() + } + + pub(crate) fn grants_of_resource(&self, resource: &ResourceIdentity) -> &'a [usize] { + self.grants_by_resource + .get(resource) + .copied() + .unwrap_or_default() + } + + pub(crate) fn grants_of_declaration(&self, declaration: &DeclarationIdentity) -> &[usize] { + self.grants_by_declaration + .get(declaration) + .map(Vec::as_slice) + .unwrap_or_default() + } + + /// Whether the usage-complete index's consumers require a placement of `identity`. + pub(crate) fn required_placement( + &self, + identity: &DeclarationIdentity, + ) -> Option<&RequiredPlacement> { + self.placements + .get_or_init(|| required_declaration_placements(&self.placement_index())) + .get(identity) + } + + /// The canonical index as placement reads it. + fn placement_index(&self) -> PlacementIndex { + let records = self.source.records; + let lookup = self.source.lookup; + PlacementIndex { + resources: lookup + .resources + .iter() + .map(|position| records.resources[*position].clone()) + .collect(), + declarations: lookup + .declarations + .iter() + .map(|position| records.declarations[*position].clone()) + .collect(), + usages: lookup + .usages + .iter() + .map(|position| { + let usage = &records.usages[*position]; + PlacementUsage { + consumer: usage.consumer.clone(), + declaration: usage.declaration.clone(), + through: usage.through.clone(), + } + }) + .collect(), + } + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs new file mode 100644 index 0000000000..50af50a8f9 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs @@ -0,0 +1,218 @@ +//! `declaration_report`: one declaration's value-safe report, with its role container. + +use std::collections::BTreeSet; + +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::scope_index::main::format_identity::format_identity; +use crate::scope_index::models::{DeclarationRecord, QualifiedIdentity, ScopeKind}; +use crate::scope_report::_helpers::context::Context; +use crate::scope_report::_helpers::paths::safe_scope_path; +use crate::scope_report::constants::{ENUM_MEMBER_PREVIEW, GROUP_DIRECTORY, PATH_SEPARATOR}; +use crate::scope_report::models::{ + DeclarationReport, DeclarationValues, MacroValues, Provenance, SourceLocation, +}; + +/// `_declaration_container`: role, visibility label, role root and bucket path. +pub(crate) type Container = (Option, &'static str, Option, Option); + +pub(crate) fn declaration_text(record: &DeclarationRecord) -> String { + format_identity(&QualifiedIdentity::Declaration(record.identity.clone())) +} + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +pub(crate) fn optional_text(value: Option<&str>) -> JsonValue { + value.map_or(JsonValue::Null, text) +} + +pub(crate) fn integer(value: i64) -> JsonValue { + JsonValue::Integer(JsonInteger::from(value)) +} + +pub(crate) fn texts(values: &[String]) -> JsonValue { + JsonValue::Array(values.iter().map(|value| text(value)).collect()) +} + +/// Python's visibility label of each scope. +fn visibility_label(scope: ScopeKind) -> &'static str { + match scope { + ScopeKind::Global => "project", + ScopeKind::Inherited => "descendant_public", + ScopeKind::Local => "exact_owner_private", + ScopeKind::Private => "model_private", + } +} + +/// `_declaration_container` of one record. +pub(crate) fn container(record: &DeclarationRecord) -> Container { + let visibility: &'static str = visibility_label(record.scope); + if record.scope == ScopeKind::Private { + return (None, visibility, None, None); + } + let role_parts: &[&str] = record.identity.kind.role_parts(); + let safe: String = safe_scope_path(&record.path); + let parts: Vec<&str> = safe.split(PATH_SEPARATOR).collect(); + let mut role_index: usize = match record.owning_path.as_deref() { + Some(owning) if !owning.is_empty() => owning.split(PATH_SEPARATOR).count(), + _ => 0, + }; + if parts.get(role_index) == Some(&GROUP_DIRECTORY) { + role_index += 1; + } + let role_end: usize = (role_index + role_parts.len()).min(parts.len()); + let bucket_end: usize = parts.len().saturating_sub(1).max(role_end); + let bucket: String = parts[role_end..bucket_end].join("/"); + ( + Some(role_parts.join("/")), + visibility, + Some(parts[..role_end].join("/")), + (!bucket.is_empty()).then_some(bucket), + ) +} + +/// The report metadata of one declaration's values and container. +fn metadata(values: Option<&DeclarationValues>, container: Container) -> Vec<(String, JsonValue)> { + let mut metadata: Vec<(String, JsonValue)> = Vec::new(); + let mut push = |key: &str, value: JsonValue| metadata.push((key.to_owned(), value)); + if let Some(values) = values { + if let Some(macro_values) = &values.macro_values { + push("parameters", texts(¯o_values.parameters)); + push( + "dependency_count", + integer(i64::try_from(macro_values.dependencies.len()).unwrap_or(i64::MAX)), + ); + } + if let Some(enum_values) = &values.enum_values { + let shown: usize = enum_values.members.len().min(ENUM_MEMBER_PREVIEW); + push("scalar_type", text(&enum_values.scalar_type)); + push("members", texts(&enum_values.members[..shown])); + push( + "member_count", + integer(i64::try_from(enum_values.members.len()).unwrap_or(i64::MAX)), + ); + push( + "members_truncated", + JsonValue::Bool(enum_values.members.len() > ENUM_MEMBER_PREVIEW), + ); + } + if let Some(constant) = &values.constant_values { + push("logical_type", text(&constant.logical_type)); + push( + "collection_kind", + optional_text(constant.collection_kind.as_deref()), + ); + push( + "item_count", + constant.item_count.map_or(JsonValue::Null, integer), + ); + push("nullable", JsonValue::Bool(constant.nullable)); + push("render_as", optional_text(constant.render_as.as_deref())); + } + } + let (role, visibility, role_root, bucket) = container; + push("role", optional_text(role.as_deref())); + push("visibility_label", text(visibility)); + push("role_root", optional_text(role_root.as_deref())); + push("bucket_path", optional_text(bucket.as_deref())); + metadata +} + +/// `required_placement`: the scope, folder and promotion impact consumers require. +fn required( + context: &Context<'_>, + record: &DeclarationRecord, +) -> (Option, Option, Vec) { + if context.usages_of_declaration(&record.identity).is_empty() + || !context.completeness().runtime_usage + { + return (None, None, Vec::new()); + } + let Some(placement) = context.required_placement(&record.identity) else { + return (None, None, Vec::new()); + }; + let current: &str = record + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(&record.ownership_root.path); + let promotion: Vec = + if placement.scope != record.scope || placement.path.as_deref() != Some(current) { + placement.consumers.clone() + } else { + Vec::new() + }; + ( + Some(placement.scope.as_str().to_owned()), + placement.path.clone(), + promotion, + ) +} + +/// The qualified identity text of each macro dependency. +fn dependency_texts(macro_values: &MacroValues) -> Vec { + macro_values + .dependencies + .iter() + .map(|identity| format_identity(&QualifiedIdentity::Declaration(identity.clone()))) + .collect() +} + +/// `declaration_report` of the record at input `position`. +pub(crate) fn declaration_report( + context: &Context<'_>, + position: usize, + visibility: Option, + inaccessible_reason: Option, +) -> DeclarationReport { + let record: &DeclarationRecord = context.declaration(position); + let values: Option<&DeclarationValues> = context.values(position); + let consumers: BTreeSet = context + .usages_of_declaration(&record.identity) + .iter() + .map(|usage| format_identity(&context.usage(*usage).consumer)) + .collect(); + let mut dependencies: Vec = values + .and_then(|values| values.macro_values.as_ref()) + .map(dependency_texts) + .unwrap_or_default(); + dependencies.sort(); + let grants: BTreeSet = context + .grants_of_declaration(&record.identity) + .iter() + .map(|grant| { + format_identity(&QualifiedIdentity::Resource( + context.grant(*grant).resource.clone(), + )) + }) + .collect(); + let (required_scope, required_path, promotion_impact) = required(context, record); + DeclarationReport { + identity: declaration_text(record), + kind: record.identity.kind.as_str().to_owned(), + name: record.identity.name.clone(), + owner: record + .identity + .owner + .as_ref() + .map(|owner| format_identity(&QualifiedIdentity::Resource(owner.clone()))), + definition: SourceLocation { + path: safe_scope_path(&record.path), + line: record.line, + column: record.column, + }, + scope: record.scope.as_str().to_owned(), + owning_path: record.owning_path.clone(), + visibility, + inaccessible_reason, + metadata: metadata(values, container(record)), + consumers: consumers.into_iter().collect(), + dependencies, + grants: grants.into_iter().collect(), + required_scope, + required_path, + promotion_impact, + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/folders.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/folders.rs new file mode 100644 index 0000000000..c19fc6d4a3 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/folders.rs @@ -0,0 +1,366 @@ +//! Declaration-definition folders: browsing their children and listing what lies beneath. + +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, QualifiedIdentity, ScopeKind, +}; +use crate::scope_report::_helpers::context::Context; +use crate::scope_report::_helpers::declarations::{declaration_report, declaration_text}; +use crate::scope_report::_helpers::paths::{ancestry, normalized}; +use crate::scope_report::_helpers::selection::count; +use crate::scope_report::_helpers::selection::{glob_matches, report_key}; +use crate::scope_report::_helpers::targets::{ + diagnostic, query, resource_matches, shown, target_text, +}; +use crate::scope_report::constants::{ + CURRENT_PATH, IDENTITY_SEPARATOR, INVALID_CURSOR, INVALID_PROSPECTIVE_PATH, LIST_SECTION, + UNKNOWN_TARGET, +}; +use crate::scope_report::models::{ + BrowseResult, DeclarationReport, DiagnosticFacts, FolderRequest, ListResult, QueryTarget, + ReportDiagnostic, ReportFailure, ReportFilters, ReportSource, ScopeFolder, ScopeSection, +}; +use sqlbuild_core::posix_path::main::posix_join::posix_join; +use sqlbuild_core::posix_path::main::posix_name::posix_name; +use sqlbuild_core::posix_path::main::posix_parent::posix_parent; +use sqlbuild_core::posix_path::main::posix_parts::posix_parts; +use std::collections::{BTreeMap, HashSet}; + +/// `_target_used_identities`: what the target uses, its diagnostics, and usage completeness. +pub(crate) fn target_used<'a>( + context: &Context<'a>, + target: Option<&QueryTarget>, + prospective: bool, +) -> (HashSet<&'a DeclarationIdentity>, Vec, bool) { + if prospective { + return (HashSet::new(), Vec::new(), false); + } + let runtime_usage: bool = context.completeness().runtime_usage; + let Some(target) = target else { + let used: HashSet<&'a DeclarationIdentity> = context + .source + .records + .usages + .iter() + .map(|usage| &usage.declaration) + .collect(); + return (used, Vec::new(), runtime_usage); + }; + let shown: String = target_text(target); + let matches: Vec = match query(context, target) { + Ok(matched) => resource_matches(matched), + Err(_) => { + let message: String = format!("Browse/list target {shown} is not a known resource"); + return ( + HashSet::new(), + vec![diagnostic(UNKNOWN_TARGET, message, None)], + false, + ); + } + }; + let [resource] = matches.as_slice() else { + let message: String = + format!("Browse/list target {shown} does not resolve to exactly one resource"); + return ( + HashSet::new(), + vec![diagnostic(UNKNOWN_TARGET, message, None)], + false, + ); + }; + let consumer: QualifiedIdentity = + QualifiedIdentity::Resource(context.resource(*resource).identity.clone()); + let used: HashSet<&'a DeclarationIdentity> = context + .usages_of_consumer(&consumer) + .iter() + .map(|usage| &context.usage(*usage).declaration) + .collect(); + (used, Vec::new(), runtime_usage) +} + +/// `_definition_folder`: globals under `global/s`, every other declaration in its folder. +pub(crate) fn definition_folder(record: &DeclarationRecord) -> String { + if record.scope != ScopeKind::Global { + return posix_parent(&record.path); + } + let source_root: &str = record + .path + .split_once('/') + .map_or(record.path.as_str(), |(root, _)| root); + let parent: String = posix_parent(&record.path); + let relative: &str = parent + .strip_prefix(source_root) + .unwrap_or(&parent) + .trim_matches('/'); + let base: String = format!("global/{}s", record.identity.kind.as_str()); + if relative.is_empty() || relative == CURRENT_PATH { + base + } else { + format!("{base}/{relative}") + } +} + +/// The direct child of `parent_parts` holding `folder`, if `folder` lies strictly below it. +pub(crate) fn direct_child(folder: &str, parent_parts: &[&str]) -> Option { + let parts: Vec<&str> = posix_parts(folder); + if parts.len() <= parent_parts.len() || parts[..parent_parts.len()] != *parent_parts { + return None; + } + Some(posix_join(&parts[..=parent_parts.len()])) +} + +/// The parts of a browsed folder; the project root has none. +pub(crate) fn folder_parts(folder: &str) -> Vec<&str> { + if folder == CURRENT_PATH { + Vec::new() + } else { + posix_parts(folder) + } +} + +/// `_folder_report`: counts beneath one child folder. +pub(crate) fn folder_report( + context: &Context<'_>, + path: String, + records: &[usize], + used: &HashSet<&DeclarationIdentity>, +) -> ScopeFolder { + let prefix: Vec<&str> = posix_parts(&path); + let mut children: HashSet = HashSet::new(); + let mut kinds: BTreeMap<&str, i64> = BTreeMap::new(); + let mut used_count: i64 = 0; + for position in records { + let record: &DeclarationRecord = context.declaration(*position); + let parts_folder: String = definition_folder(record); + let parts: Vec<&str> = posix_parts(&parts_folder); + if parts.len() > prefix.len() { + children.insert(posix_join(&parts[..=prefix.len()])); + } + *kinds.entry(record.identity.kind.as_str()).or_default() += 1; + used_count += i64::from(used.contains(&record.identity)); + } + ScopeFolder { + name: posix_name(&path).to_owned(), + path, + descendant_count: count(records.len()), + child_count: count(children.len()), + used_count, + kind_counts: kinds + .into_iter() + .map(|(kind, total)| (kind.to_owned(), total)) + .collect(), + } +} + +/// The browse result of `request` over one lookup. +pub(crate) fn browsing(source: ReportSource<'_>, request: &FolderRequest) -> BrowseResult { + let context: Context<'_> = Context::new(source); + let Ok(folder) = normalized(&request.folder) else { + let message: String = format!( + "Folder {} is not a project-relative declaration folder", + shown(&request.folder) + ); + return BrowseResult { + folder: request.folder.clone(), + folders: Vec::new(), + diagnostics: vec![ReportDiagnostic::Raised(diagnostic( + INVALID_PROSPECTIVE_PATH, + message, + None, + ))], + complete: false, + }; + }; + let (used, target_diagnostics, usage_complete) = + target_used(&context, request.target.as_ref(), request.prospective); + let parent: Vec<&str> = folder_parts(&folder); + let mut children: BTreeMap> = BTreeMap::new(); + for position in context.canonical_declarations() { + let defined: String = definition_folder(context.declaration(*position)); + if let Some(child) = direct_child(&defined, &parent) { + children.entry(child).or_default().push(*position); + } + } + let folders: Vec = children + .into_iter() + .map(|(path, records)| folder_report(&context, path, &records, &used)) + .collect(); + let complete: bool = + context.completeness().discovery && usage_complete && target_diagnostics.is_empty(); + BrowseResult { + folder, + folders, + diagnostics: (0..source.lookup.diagnostics.len()) + .map(ReportDiagnostic::Index) + .chain(target_diagnostics.into_iter().map(ReportDiagnostic::Raised)) + .collect(), + complete, + } +} + +/// `ScopeReportFilters(globals="all")`. +fn default_filters() -> ReportFilters { + ReportFilters { + sections: vec![ + "available".to_owned(), + "used".to_owned(), + "relationship_scope".to_owned(), + ], + include_nearby: false, + defined_under: None, + kinds: Vec::new(), + glob: None, + used_only: false, + dependency_depth: 0, + sort: "identity".to_owned(), + cursor: None, + page_size: 100, + nearby_depth: 1, + globals: "all".to_owned(), + } +} + +/// The records beneath `under`; any path `normalize_path` rejects empties them. +fn defined_beneath(context: &Context<'_>, records: Vec, under: &str) -> Vec { + let Ok(under) = normalized(under) else { + return Vec::new(); + }; + let mut kept: Vec = Vec::new(); + for position in records { + match ancestry(&context.declaration(position).path, &under) { + Ok(true) => kept.push(position), + Ok(false) => {} + Err(_) => return Vec::new(), + } + } + kept +} + +/// The list result of `request`, or the path error Python raises. +pub(crate) fn listing( + source: ReportSource<'_>, + request: &FolderRequest, +) -> Result { + let context: Context<'_> = Context::new(source); + let filters: ReportFilters = request.filters.clone().unwrap_or_else(default_filters); + let Ok(folder) = normalized(&request.folder) else { + let message: String = format!( + "Folder {} is not a project-relative declaration folder", + shown(&request.folder) + ); + return Ok(ListResult { + folder: request.folder.clone(), + declarations: Vec::new(), + section: ScopeSection { + name: LIST_SECTION.to_owned(), + total: 0, + returned: 0, + collapsed: false, + collapsed_count: 0, + truncated: false, + complete: false, + next_cursor: None, + cursor: None, + page_size: filters.page_size, + }, + diagnostics: vec![ReportDiagnostic::Raised(diagnostic( + INVALID_PROSPECTIVE_PATH, + message, + None, + ))], + }); + }; + let (used, target_diagnostics, usage_complete) = + target_used(&context, request.target.as_ref(), request.prospective); + let mut records: Vec = Vec::new(); + for position in context.canonical_declarations() { + let defined: String = definition_folder(context.declaration(*position)); + if ancestry(&defined, &folder).map_err(ReportFailure::Path)? { + records.push(*position); + } + } + if !filters.kinds.is_empty() { + records.retain(|position| { + let kind: &str = context.declaration(*position).identity.kind.as_str(); + filters.kinds.iter().any(|selected| selected == kind) + }); + } + if filters.used_only { + records.retain(|position| used.contains(&context.declaration(*position).identity)); + } + if let Some(under) = &filters.defined_under { + records = defined_beneath(&context, records, under); + } + if let Some(glob) = &filters.glob { + records.retain(|position| glob_matches(context.declaration(*position), glob)); + } + records.sort_by_cached_key(|position| report_key(&context, *position)); + let mut cursor_diagnostic: Option = None; + let mut cursor: Option = filters.cursor.clone(); + if let Some(requested) = cursor.as_deref() + && !requested.contains(IDENTITY_SEPARATOR) + { + let candidates: Vec = records + .iter() + .copied() + .filter(|position| context.declaration(*position).identity.name == requested) + .collect(); + cursor = match candidates.as_slice() { + [only] => Some(declaration_text(context.declaration(*only))), + _ => { + let message: String = format!("Cursor {} is invalid", shown(requested)); + cursor_diagnostic = Some(diagnostic(INVALID_CURSOR, message, None)); + None + } + }; + } + let mut start: usize = 0; + if let Some(cursor) = cursor.as_deref() { + let positions: Vec = records + .iter() + .enumerate() + .filter(|(_, position)| declaration_text(context.declaration(**position)) == cursor) + .map(|(index, _)| index) + .collect(); + match positions.as_slice() { + [index] => start = index + 1, + _ => { + let message: String = format!("Cursor {} is invalid", shown(cursor)); + cursor_diagnostic = Some(diagnostic(INVALID_CURSOR, message, None)); + } + } + } + let size: usize = usize::try_from(filters.page_size.max(0)).unwrap_or(usize::MAX); + let end: usize = start.saturating_add(size).min(records.len()); + let selected: &[usize] = records.get(start..end).unwrap_or_default(); + let reports: Vec = selected + .iter() + .map(|position| declaration_report(&context, *position, None, None)) + .collect(); + let truncated: bool = start + selected.len() < records.len(); + let section: ScopeSection = ScopeSection { + name: LIST_SECTION.to_owned(), + total: count(records.len()), + returned: count(reports.len()), + collapsed: false, + collapsed_count: 0, + truncated, + complete: context.completeness().discovery + && usage_complete + && target_diagnostics.is_empty(), + next_cursor: reports + .last() + .filter(|_| truncated) + .map(|report| report.identity.clone()), + cursor, + page_size: count(size), + }; + Ok(ListResult { + folder, + declarations: reports, + section, + diagnostics: (0..source.lookup.diagnostics.len()) + .map(ReportDiagnostic::Index) + .chain(target_diagnostics.into_iter().map(ReportDiagnostic::Raised)) + .chain(cursor_diagnostic.into_iter().map(ReportDiagnostic::Raised)) + .collect(), + }) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/json.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/json.rs new file mode 100644 index 0000000000..712603ea66 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/json.rs @@ -0,0 +1,249 @@ +//! JSON objects of report results, as `serialize_result` walks the report dataclasses. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::scope_report::_helpers::declarations::{integer, optional_text, text, texts}; +use crate::scope_report::_helpers::paths::safe_scope_path; +use crate::scope_report::constants::ESCAPE_CHARACTER; +use crate::scope_report::models::{ + DeclarationReport, DiagnosticFacts, Explanation, MovePreview, Provenance, ReportDiagnostic, + ReportFailure, ReportFilters, ResourceReport, ScopeChainEntry, ScopeFolder, ScopeSection, +}; + +/// One JSON object from its fields in order. +pub(crate) fn object(fields: Vec<(&str, JsonValue)>) -> JsonValue { + JsonValue::Object( + fields + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect(), + ) +} + +/// `json.dumps(value, sort_keys=True, ensure_ascii=True, separators=(",", ":"))`. +pub(crate) fn canonical_json(value: &JsonValue) -> Result { + let mut options: StdlibJsonOptions = StdlibJsonOptions::new(None).with_separators(",", ":"); + options.sort_keys = true; + dumps(value, &JsonDialect::Stdlib(options)) + .map_err(|error| ReportFailure::Serialization(format!("{error:?}"))) +} + +fn optional_integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, integer) +} + +/// `diagnostic_projection`. +pub(crate) fn diagnostic_json(diagnostic: &DiagnosticFacts) -> JsonValue { + object(vec![ + ("code", text(&diagnostic.code)), + ( + "message", + JsonValue::String(diagnostic.message.replace(ESCAPE_CHARACTER, "")), + ), + ("severity", text(&diagnostic.severity)), + ( + "path", + diagnostic + .path + .as_deref() + .map_or(JsonValue::Null, |path| text(&safe_scope_path(path))), + ), + ("line", optional_integer(diagnostic.line)), + ("column", optional_integer(diagnostic.column)), + ( + "declaration", + optional_text(diagnostic.declaration.as_deref()), + ), + ("resource", optional_text(diagnostic.resource.as_deref())), + ]) +} + +pub(crate) fn diagnostics_json( + diagnostics: &[ReportDiagnostic], +) -> Result { + let mut values: Vec = Vec::new(); + for diagnostic in diagnostics { + match diagnostic { + ReportDiagnostic::Raised(facts) => values.push(diagnostic_json(facts)), + ReportDiagnostic::Index(_) => { + return Err(ReportFailure::Serialization( + "an index diagnostic reached serialization without its facts".to_owned(), + )); + } + } + } + Ok(JsonValue::Array(values)) +} + +fn provenance_json(provenance: Option<&Provenance>) -> JsonValue { + provenance.map_or(JsonValue::Null, |provenance| { + object(vec![ + ("reason", text(&provenance.reason)), + ("through", optional_text(provenance.through.as_deref())), + ]) + }) +} + +pub(crate) fn declaration_json(report: &DeclarationReport) -> JsonValue { + object(vec![ + ("identity", text(&report.identity)), + ("kind", text(&report.kind)), + ("name", text(&report.name)), + ("owner", optional_text(report.owner.as_deref())), + ( + "definition", + object(vec![ + ("path", text(&report.definition.path)), + ("line", integer(report.definition.line)), + ("column", integer(report.definition.column)), + ]), + ), + ("scope", text(&report.scope)), + ("owning_path", optional_text(report.owning_path.as_deref())), + ("visibility", provenance_json(report.visibility.as_ref())), + ( + "inaccessible_reason", + optional_text(report.inaccessible_reason.as_deref()), + ), + ("metadata", JsonValue::Object(report.metadata.clone())), + ("consumers", texts(&report.consumers)), + ("dependencies", texts(&report.dependencies)), + ("grants", texts(&report.grants)), + ( + "required_scope", + optional_text(report.required_scope.as_deref()), + ), + ( + "required_path", + optional_text(report.required_path.as_deref()), + ), + ("promotion_impact", texts(&report.promotion_impact)), + ]) +} + +pub(crate) fn declarations_json(reports: &[DeclarationReport]) -> JsonValue { + JsonValue::Array(reports.iter().map(declaration_json).collect()) +} + +pub(crate) fn section_json(section: &ScopeSection) -> JsonValue { + object(vec![ + ("name", text(§ion.name)), + ("total", integer(section.total)), + ("returned", integer(section.returned)), + ("collapsed", JsonValue::Bool(section.collapsed)), + ("collapsed_count", integer(section.collapsed_count)), + ("truncated", JsonValue::Bool(section.truncated)), + ("complete", JsonValue::Bool(section.complete)), + ("next_cursor", optional_text(section.next_cursor.as_deref())), + ("cursor", optional_text(section.cursor.as_deref())), + ("page_size", integer(section.page_size)), + ]) +} + +pub(crate) fn resource_json(resource: &ResourceReport) -> JsonValue { + object(vec![ + ("target", text(&resource.target)), + ("identity", optional_text(resource.identity.as_deref())), + ("path", optional_text(resource.path.as_deref())), + ("prospective", JsonValue::Bool(resource.prospective)), + ("directory", JsonValue::Bool(resource.directory)), + ("duplicate_count", integer(resource.duplicate_count)), + ]) +} + +pub(crate) fn chain_json(entries: &[ScopeChainEntry]) -> JsonValue { + JsonValue::Array( + entries + .iter() + .map(|entry| { + object(vec![ + ("kind", text(&entry.kind)), + ("path", text(&entry.path)), + ("declaration_count", integer(entry.declaration_count)), + ]) + }) + .collect(), + ) +} + +pub(crate) fn filters_json(filters: &ReportFilters) -> JsonValue { + object(vec![ + ("sections", texts(&filters.sections)), + ("include_nearby", JsonValue::Bool(filters.include_nearby)), + ( + "defined_under", + optional_text(filters.defined_under.as_deref()), + ), + ("kinds", texts(&filters.kinds)), + ("glob", optional_text(filters.glob.as_deref())), + ("used_only", JsonValue::Bool(filters.used_only)), + ("dependency_depth", integer(filters.dependency_depth)), + ("sort", text(&filters.sort)), + ("cursor", optional_text(filters.cursor.as_deref())), + ("page_size", integer(filters.page_size)), + ("nearby_depth", integer(filters.nearby_depth)), + ("globals", text(&filters.globals)), + ]) +} + +pub(crate) fn explanation_json(explanation: Option<&Explanation>) -> JsonValue { + explanation.map_or(JsonValue::Null, |explanation| { + object(vec![ + ( + "declaration", + explanation + .declaration + .as_ref() + .map_or(JsonValue::Null, declaration_json), + ), + ("complete", JsonValue::Bool(explanation.complete)), + ]) + }) +} + +pub(crate) fn move_json(preview: Option<&MovePreview>) -> JsonValue { + preview.map_or(JsonValue::Null, |preview| { + object(vec![ + ("resource", text(&preview.resource)), + ("destination", text(&preview.destination)), + ( + "new_ownership_root", + optional_text(preview.new_ownership_root.as_deref()), + ), + ("retained", declarations_json(&preview.retained)), + ("gained", declarations_json(&preview.gained)), + ("lost", declarations_json(&preview.lost)), + ("invalidated_usages", texts(&preview.invalidated_usages)), + ( + "private_retained", + declarations_json(&preview.private_retained), + ), + ( + "relationship_retained", + declarations_json(&preview.relationship_retained), + ), + ("complete", JsonValue::Bool(preview.complete)), + ]) + }) +} + +pub(crate) fn folder_json(folder: &ScopeFolder) -> JsonValue { + object(vec![ + ("path", text(&folder.path)), + ("name", text(&folder.name)), + ("descendant_count", integer(folder.descendant_count)), + ("child_count", integer(folder.child_count)), + ("used_count", integer(folder.used_count)), + ( + "kind_counts", + JsonValue::Object( + folder + .kind_counts + .iter() + .map(|(kind, count)| (kind.clone(), integer(*count))) + .collect(), + ), + ), + ]) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/mod.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/mod.rs new file mode 100644 index 0000000000..9442456a97 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod context; +pub(crate) mod declarations; +pub(crate) mod folders; +pub(crate) mod json; +pub(crate) mod moves; +pub(crate) mod paths; +pub(crate) mod projection; +pub(crate) mod report; +pub(crate) mod selection; +pub(crate) mod targets; diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/moves.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/moves.rs new file mode 100644 index 0000000000..66304077ca --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/moves.rs @@ -0,0 +1,208 @@ +//! The visibility deltas `preview_scope_move` reports for one resource move. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use crate::scope_index::main::classify_report_resource::classify_report_resource; +use crate::scope_index::main::format_identity::format_identity; +use crate::scope_index::models::{ + DeclarationIdentity, OwnershipRoot, QualifiedIdentity, ReportClassification, ResourceRecord, + ScopeKind, +}; +use crate::scope_report::_helpers::context::Context; +use crate::scope_report::_helpers::declarations::declaration_report; +use crate::scope_report::_helpers::paths::{ancestry, normalized, resource_path_is_valid}; +use crate::scope_report::_helpers::targets::{ + diagnostic, ownership_roots, query, resource_matches, target_text, +}; +use crate::scope_report::constants::{INVALID_PROSPECTIVE_PATH, UNKNOWN_TARGET}; +use crate::scope_report::models::{ + DeclarationReport, DiagnosticFacts, MovePreview, MoveRequest, Provenance, ReportFailure, + ReportSource, +}; + +/// Declarations at input positions with how each is visible. +type Visible = Vec<(usize, Provenance)>; + +/// Visible declarations by qualified identity text, later records replacing earlier ones. +type ByIdentity<'a> = BTreeMap; + +fn invalid_destination(destination: &str) -> DiagnosticFacts { + diagnostic( + INVALID_PROSPECTIVE_PATH, + format!( + "Move destination {destination} is invalid, outside its authored resource root, \ + or already occupied" + ), + None, + ) +} + +fn qualified(identity: &DeclarationIdentity) -> String { + format_identity(&QualifiedIdentity::Declaration(identity.clone())) +} + +/// Direct or relationship visibility keyed by identity. +fn keyed<'a>(context: &Context<'a>, values: Visible) -> ByIdentity<'a> { + let mut keyed: ByIdentity<'a> = BTreeMap::new(); + for (position, provenance) in values { + let identity: &'a DeclarationIdentity = &context.declaration(position).identity; + keyed.insert(qualified(identity), (identity, position, provenance)); + } + keyed +} + +/// The classification's direct and relationship visibility. +fn visible(classification: ReportClassification) -> (Visible, Visible) { + let direct: Visible = classification + .direct + .into_iter() + .map(|(position, reason)| { + let provenance = Provenance { + reason: reason.as_str().to_owned(), + through: None, + }; + (position, provenance) + }) + .collect(); + let relationships: Visible = classification + .relationships + .into_iter() + .map(|entry| { + let provenance = Provenance { + reason: entry.reason.as_str().to_owned(), + through: Some(entry.through), + }; + (entry.position, provenance) + }) + .collect(); + (direct, relationships) +} + +/// Reports of `values` whose identity text satisfies `keep`, in identity order. +fn reports( + context: &Context<'_>, + values: &ByIdentity<'_>, + keep: impl Fn(&str) -> bool, +) -> Vec { + values + .iter() + .filter(|(text, _)| keep(text)) + .map(|(_, (_, position, provenance))| { + declaration_report(context, *position, Some(provenance.clone()), None) + }) + .collect() +} + +/// The roots that could own `source` at `path`. +fn destination_roots( + context: &Context<'_>, + source: &ResourceRecord, + path: &str, +) -> Result, ReportFailure> { + let mut roots: Vec = Vec::new(); + for root in ownership_roots(context) { + if root.resource_kind != Some(source.identity.kind) { + continue; + } + if ancestry(path, &root.path).map_err(ReportFailure::Path)? + && resource_path_is_valid(path, &root) + && !roots.contains(&root) + { + roots.push(root); + } + } + Ok(roots) +} + +/// The move preview of `request`, or why it cannot be previewed. +pub(crate) fn move_preview( + source: ReportSource<'_>, + request: &MoveRequest, +) -> Result<(Option, Vec), ReportFailure> { + let context: Context<'_> = Context::new(source); + let matches: Vec = query(&context, &request.resource) + .map(resource_matches) + .unwrap_or_default(); + let [position] = matches.as_slice() else { + let message: String = format!( + "Move source {} does not resolve to exactly one resource", + target_text(&request.resource) + ); + return Ok((None, vec![diagnostic(UNKNOWN_TARGET, message, None)])); + }; + let moving: &ResourceRecord = context.resource(*position); + let Ok(path) = normalized(&request.destination) else { + return Ok((None, vec![invalid_destination(&request.destination)])); + }; + let roots: Vec = destination_roots(&context, moving, &path)?; + let occupied: bool = !context.resources_at(&path).is_empty() && path != moving.path; + let ([root], false) = (roots.as_slice(), occupied) else { + return Ok((None, vec![invalid_destination(&request.destination)])); + }; + let moved: ResourceRecord = ResourceRecord { + identity: moving.identity.clone(), + path: path.clone(), + ownership_root: root.clone(), + }; + let (old_direct, old_relationships) = visible( + classify_report_resource(source.records, source.lookup, moving) + .map_err(ReportFailure::Path)?, + ); + let (new_direct, new_relationships) = visible( + classify_report_resource(source.records, source.lookup, &moved) + .map_err(ReportFailure::Path)?, + ); + let old: ByIdentity<'_> = keyed(&context, old_direct); + let new: ByIdentity<'_> = keyed(&context, new_direct); + let consumer: QualifiedIdentity = QualifiedIdentity::Resource(moving.identity.clone()); + let direct_usages: HashSet = context + .usages_of_consumer(&consumer) + .iter() + .map(|usage| context.usage(*usage)) + .filter(|usage| usage.through.is_none()) + .map(|usage| qualified(&usage.declaration)) + .collect(); + let old_related: ByIdentity<'_> = keyed(&context, old_relationships); + let new_related: HashSet = keyed(&context, new_relationships).into_keys().collect(); + let private: HashMap = context + .canonical_declarations() + .iter() + .map(|position| context.declaration(*position)) + .filter(|record| { + record.scope == ScopeKind::Private + && record.identity.owner.as_ref() == Some(&moving.identity) + }) + .map(|record| (qualified(&record.identity), &record.identity)) + .collect(); + let mut private_order: Vec<(&String, &&DeclarationIdentity)> = private.iter().collect(); + private_order.sort_by(|left, right| left.0.cmp(right.0)); + let private_retained: Vec = private_order + .into_iter() + .flat_map(|(_, identity)| context.declarations_of(identity).iter()) + .map(|position| declaration_report(&context, *position, None, None)) + .collect(); + let completeness = context.completeness(); + Ok(( + Some(MovePreview { + resource: format_identity(&consumer), + destination: path, + new_ownership_root: Some(root.path.clone()), + retained: reports(&context, &old, |text| new.contains_key(text)), + gained: reports(&context, &new, |text| !old.contains_key(text)), + lost: reports(&context, &old, |text| !new.contains_key(text)), + invalidated_usages: old + .keys() + .filter(|text| !new.contains_key(*text) && direct_usages.contains(*text)) + .cloned() + .collect(), + private_retained, + relationship_retained: reports(&context, &old_related, |text| { + new_related.contains(text) + }), + complete: completeness.static_visibility + && completeness.runtime_usage + && completeness.relationships, + }), + Vec::new(), + )) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/paths.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/paths.rs new file mode 100644 index 0000000000..9ab5e01c12 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/paths.rs @@ -0,0 +1,72 @@ +//! Python's scope path relations, `PurePosixPath` reads and resource path rules of the reports. + +use sqlbuild_core::posix_path::main::posix_parts::posix_parts; +use sqlbuild_core::posix_path::main::posix_suffix::posix_suffix; +use sqlbuild_core::text::main::active_python_text::active_python_text; + +use crate::scope_index::main::normalize_scope_path::normalize_scope_path; +use crate::scope_index::models::{OwnershipRoot, RejectedPath, ResourceKind}; +use crate::scope_report::constants::{ + CURRENT_PATH, INVALID_PATH_TEXT, PATH_SEPARATOR, SQL_SUFFIX, WINDOWS_PATH_SEPARATOR, + YAML_SUFFIX, YML_SUFFIX, +}; + +/// Python's `normalize_path`, or the rejection `InvalidScopePathError` reports. +pub(crate) fn normalized(path: &str) -> Result { + normalize_scope_path(path).map_err(|failure| RejectedPath { + failure, + path: path.to_owned(), + }) +} + +/// `safe_scope_path`: the normalized path, without leaking an invalid one. +pub(crate) fn safe_scope_path(path: &str) -> String { + normalize_scope_path(path).unwrap_or_else(|_| INVALID_PATH_TEXT.to_owned()) +} + +/// Python's `ancestry` over normalized paths. +pub(crate) fn ancestry(path: &str, ancestor: &str) -> Result { + let path: String = normalized(path)?; + let ancestor: String = normalized(ancestor)?; + if ancestor == CURRENT_PATH { + return Ok(true); + } + let path_parts: Vec<&str> = path.split(PATH_SEPARATOR).collect(); + let ancestor_parts: Vec<&str> = ancestor.split(PATH_SEPARATOR).collect(); + Ok(path_parts.len() >= ancestor_parts.len() + && path_parts[..ancestor_parts.len()] == ancestor_parts[..]) +} + +/// `len(PurePosixPath(path).parts)`. +pub(crate) fn part_count(path: &str) -> usize { + posix_parts(path).len() +} + +/// `_looks_like_path`: a separator or a `PurePosixPath` suffix. +pub(crate) fn looks_like_path(value: &str) -> bool { + value.contains(PATH_SEPARATOR) + || value.contains(WINDOWS_PATH_SEPARATOR) + || !posix_suffix(active_python_text(), value).is_empty() +} + +/// `_default_suffix`: sources are YAML, every other resource SQL. +pub(crate) fn default_suffix(root: &OwnershipRoot) -> &'static str { + if root.resource_kind == Some(ResourceKind::Source) { + YML_SUFFIX + } else { + SQL_SUFFIX + } +} + +/// `_resource_path_is_valid`: below the root, with the root's authored suffix in any case. +pub(crate) fn resource_path_is_valid(path: &str, root: &OwnershipRoot) -> bool { + if path == root.path { + return false; + } + let suffix: String = posix_suffix(active_python_text(), path).to_ascii_lowercase(); + if root.resource_kind == Some(ResourceKind::Source) { + suffix == YML_SUFFIX || suffix == YAML_SUFFIX + } else { + suffix == SQL_SUFFIX + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs new file mode 100644 index 0000000000..8cbe9a0a09 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs @@ -0,0 +1,341 @@ +//! The value-free metadata projection of one full scope index. + +use sqlbuild_core::json::models::JsonValue; + +use crate::scope_index::main::format_identity::format_identity; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, QualifiedIdentity, ResourceIdentity, +}; +use crate::scope_report::_helpers::declarations::{container, integer, optional_text, text, texts}; +use crate::scope_report::_helpers::json::{diagnostic_json, object}; +use crate::scope_report::_helpers::paths::safe_scope_path; +use crate::scope_report::constants::METADATA_SCHEMA_VERSION; +use crate::scope_report::models::{Completeness, DeclarationValues, ProjectionIndex}; + +fn resource_key(identity: &ResourceIdentity) -> String { + format_identity(&QualifiedIdentity::Resource(identity.clone())) +} + +fn declaration_key(identity: &DeclarationIdentity) -> String { + format_identity(&QualifiedIdentity::Declaration(identity.clone())) +} + +fn optional_key(identity: Option<&QualifiedIdentity>) -> String { + identity.map(format_identity).unwrap_or_default() +} + +/// Positions of `count` items in the stable order of `key`. +fn sorted(count: usize, key: impl Fn(usize) -> K) -> Vec { + let mut positions: Vec = (0..count).collect(); + positions.sort_by_cached_key(|position| key(*position)); + positions +} + +fn declaration_metadata(values: Option<&DeclarationValues>) -> JsonValue { + let mut metadata: Vec<(&str, JsonValue)> = Vec::new(); + let Some(values) = values else { + return object(metadata); + }; + if let Some(macro_values) = &values.macro_values { + let mut dependencies: Vec = macro_values + .dependencies + .iter() + .map(declaration_key) + .collect(); + dependencies.sort(); + metadata.push(( + "macro", + object(vec![ + ("parameters", texts(¯o_values.parameters)), + ("dependencies", texts(&dependencies)), + ("source_digest", text(¯o_values.source_digest)), + ]), + )); + } + if let Some(enum_values) = &values.enum_values { + let members: Vec = enum_values + .members + .iter() + .map(|member| object(vec![("name", text(member))])) + .collect(); + metadata.push(( + "enum", + object(vec![ + ("members", JsonValue::Array(members)), + ("scalar_type", text(&enum_values.scalar_type)), + ]), + )); + } + if let Some(constant) = &values.constant_values { + metadata.push(( + "constant", + object(vec![ + ("logical_type", text(&constant.logical_type)), + ( + "collection_kind", + optional_text(constant.collection_kind.as_deref()), + ), + ( + "item_count", + constant.item_count.map_or(JsonValue::Null, integer), + ), + ("nullable", JsonValue::Bool(constant.nullable)), + ("render_as", optional_text(constant.render_as.as_deref())), + ]), + )); + } + object(metadata) +} + +fn declaration_projection( + record: &DeclarationRecord, + values: Option<&DeclarationValues>, +) -> JsonValue { + let (role, visibility, role_root, bucket) = container(record); + object(vec![ + ("identity", text(&declaration_key(&record.identity))), + ("kind", text(record.identity.kind.as_str())), + ("name", text(&record.identity.name)), + ( + "owner", + record + .identity + .owner + .as_ref() + .map_or(JsonValue::Null, |owner| text(&resource_key(owner))), + ), + ("path", text(&safe_scope_path(&record.path))), + ("line", integer(record.line)), + ("column", integer(record.column)), + ("scope", text(record.scope.as_str())), + ("role", optional_text(role.as_deref())), + ("visibility", text(visibility)), + ("role_root", optional_text(role_root.as_deref())), + ("bucket_path", optional_text(bucket.as_deref())), + ( + "ownership_root", + text(&safe_scope_path(&record.ownership_root.path)), + ), + ( + "owning_path", + record + .owning_path + .as_deref() + .map_or(JsonValue::Null, |path| text(&safe_scope_path(path))), + ), + ("metadata", declaration_metadata(values)), + ]) +} + +fn completeness_json(completeness: Completeness) -> JsonValue { + object(vec![ + ("discovery", JsonValue::Bool(completeness.discovery)), + ( + "static_visibility", + JsonValue::Bool(completeness.static_visibility), + ), + ("runtime_usage", JsonValue::Bool(completeness.runtime_usage)), + ("relationships", JsonValue::Bool(completeness.relationships)), + ("placement", JsonValue::Bool(completeness.placement)), + ( + "promotion_impact", + JsonValue::Bool(completeness.promotion_impact), + ), + ]) +} + +fn all_complete(completeness: Completeness) -> bool { + completeness.discovery + && completeness.static_visibility + && completeness.runtime_usage + && completeness.relationships + && completeness.placement + && completeness.promotion_impact +} + +/// The projection Python's `build_projection` returns, as a JSON object. +pub(crate) fn projection(index: &ProjectionIndex) -> JsonValue { + let roots: Vec = sorted(index.roots.len(), |position| { + let root = &index.roots[position]; + ( + root.path.clone(), + root.resource_kind + .map(|kind| kind.as_str()) + .unwrap_or_default(), + ) + }) + .into_iter() + .map(|position| { + let root = &index.roots[position]; + object(vec![ + ("path", text(&safe_scope_path(&root.path))), + ( + "resource_kind", + optional_text(root.resource_kind.map(|kind| kind.as_str())), + ), + ]) + }) + .collect(); + let resources: Vec = sorted(index.resources.len(), |position| { + let record = &index.resources[position]; + (resource_key(&record.identity), record.path.clone()) + }) + .into_iter() + .map(|position| { + let record = &index.resources[position]; + object(vec![ + ("identity", text(&resource_key(&record.identity))), + ("kind", text(record.identity.kind.as_str())), + ("name", text(&record.identity.name)), + ("path", text(&safe_scope_path(&record.path))), + ( + "ownership_root", + text(&safe_scope_path(&record.ownership_root.path)), + ), + ( + "ownership_root_kind", + optional_text( + record + .ownership_root + .resource_kind + .map(|kind| kind.as_str()), + ), + ), + ]) + }) + .collect(); + let declarations: Vec = sorted(index.declarations.len(), |position| { + let record = &index.declarations[position]; + ( + declaration_key(&record.identity), + record.path.clone(), + record.line, + record.column, + ) + }) + .into_iter() + .map(|position| { + declaration_projection( + &index.declarations[position], + index.values.declarations.get(position), + ) + }) + .collect(); + let grants: Vec = sorted(index.grants.len(), |position| { + let grant = &index.grants[position]; + ( + resource_key(&grant.resource), + declaration_key(&grant.declaration), + format_identity(&grant.through), + grant.kind.as_str(), + ) + }) + .into_iter() + .map(|position| { + let grant = &index.grants[position]; + object(vec![ + ("resource", text(&resource_key(&grant.resource))), + ("declaration", text(&declaration_key(&grant.declaration))), + ("through", text(&format_identity(&grant.through))), + ("kind", text(grant.kind.as_str())), + ]) + }) + .collect(); + let usages: Vec = sorted(index.usages.len(), |position| { + let usage = &index.usages[position]; + ( + format_identity(&usage.consumer), + declaration_key(&usage.declaration), + usage.kind.clone(), + optional_key(usage.through.as_ref()), + usage.enum_member.clone().unwrap_or_default(), + ) + }) + .into_iter() + .map(|position| { + let usage = &index.usages[position]; + object(vec![ + ("consumer", text(&format_identity(&usage.consumer))), + ("declaration", text(&declaration_key(&usage.declaration))), + ("kind", text(&usage.kind)), + ( + "through", + usage + .through + .as_ref() + .map_or(JsonValue::Null, |through| text(&format_identity(through))), + ), + ("enum_member", optional_text(usage.enum_member.as_deref())), + ]) + }) + .collect(); + let visibility: Vec = sorted(index.visibility.len(), |position| { + let fact = &index.visibility[position]; + ( + resource_key(&fact.resource), + declaration_key(&fact.declaration), + fact.reason.clone(), + optional_key(fact.through.as_ref()), + ) + }) + .into_iter() + .map(|position| { + let fact = &index.visibility[position]; + object(vec![ + ("resource", text(&resource_key(&fact.resource))), + ("declaration", text(&declaration_key(&fact.declaration))), + ("reason", text(&fact.reason)), + ( + "through", + fact.through + .as_ref() + .map_or(JsonValue::Null, |through| text(&format_identity(through))), + ), + ]) + }) + .collect(); + let inaccessible: Vec = sorted(index.inaccessible.len(), |position| { + let fact = &index.inaccessible[position]; + ( + resource_key(&fact.resource), + declaration_key(&fact.declaration), + fact.reason.clone(), + ) + }) + .into_iter() + .map(|position| { + let fact = &index.inaccessible[position]; + object(vec![ + ("resource", text(&resource_key(&fact.resource))), + ("declaration", text(&declaration_key(&fact.declaration))), + ("reason", text(&fact.reason)), + ]) + }) + .collect(); + let diagnostics: Vec = sorted(index.diagnostics.len(), |position| { + let diagnostic = &index.diagnostics[position]; + ( + diagnostic.path.clone().unwrap_or_default(), + diagnostic.line.unwrap_or(0), + diagnostic.column.unwrap_or(0), + diagnostic.code.clone(), + ) + }) + .into_iter() + .map(|position| diagnostic_json(&index.diagnostics[position])) + .collect(); + let completeness: Completeness = index.values.completeness; + object(vec![ + ("schema_version", integer(METADATA_SCHEMA_VERSION)), + ("ownership_roots", JsonValue::Array(roots)), + ("resources", JsonValue::Array(resources)), + ("declarations", JsonValue::Array(declarations)), + ("grants", JsonValue::Array(grants)), + ("usages", JsonValue::Array(usages)), + ("visibility", JsonValue::Array(visibility)), + ("inaccessible", JsonValue::Array(inaccessible)), + ("complete", JsonValue::Bool(all_complete(completeness))), + ("completeness", completeness_json(completeness)), + ("diagnostics", JsonValue::Array(diagnostics)), + ]) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/report.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/report.rs new file mode 100644 index 0000000000..d395695545 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/report.rs @@ -0,0 +1,396 @@ +//! The phases of `query_scope_report`: resolve the target, select its sections, page them. + +use std::collections::{HashMap, HashSet}; + +use crate::scope_index::main::classify_report_resource::classify_report_resource; +use crate::scope_index::models::{ + DeclarationIdentity, QualifiedIdentity, ReportClassification, ResourceRecord, ScopeKind, +}; +use crate::scope_report::_helpers::context::Context; +use crate::scope_report::_helpers::declarations::declaration_report; +use crate::scope_report::_helpers::selection::{PageSpec, count, page, resolve_cursor}; +use crate::scope_report::_helpers::selection::{ + Selected, expand_dependencies, filtered, nearby, report_key, scope_chain, +}; +use crate::scope_report::_helpers::targets::explain_declaration; +use crate::scope_report::_helpers::targets::{ + ExistingTarget, diagnostic, existing_target, prospective_resource, +}; +use crate::scope_report::constants::{ + AVAILABLE_SECTION, GLOBAL_SUMMARY_POLICY, GLOBAL_USED_POLICY, INCOMPLETE_USAGE, NEARBY_SECTION, + PROSPECTIVE_USAGE_MESSAGE, RELATIONSHIP_SECTION, USED_SECTION, +}; +use crate::scope_report::models::{ + DeclarationReport, DeclarationTarget, DiagnosticFacts, Explanation, Provenance, QueryTarget, + ReportDiagnostic, ReportFailure, ReportFilters, ReportRequest, ResourceReport, ScopeReport, + ScopeSection, +}; + +/// The record a report resolved, or the report it ends with when there is none. +pub(crate) enum Resolved { + Record(ResourceReport, ResourceRecord, Vec), + Done( + ResourceReport, + Option>, + Vec, + ), +} + +/// What a resource uses: everything, directly, and through relationships. +struct Used<'a> { + all: HashSet<&'a DeclarationIdentity>, + direct: HashSet<&'a DeclarationIdentity>, + related: HashSet<&'a DeclarationIdentity>, +} + +/// The filtered sections of one report, before paging. +struct Sections { + direct: Selected, + used: Selected, + relationships: Selected, + nearby: Selected, + collapsed: i64, +} + +/// `_existing_target` or `_prospective_resource` of the request. +pub(crate) fn resolve( + context: &Context<'_>, + request: &ReportRequest, +) -> Result { + if let Some(at) = &request.at { + let (resource, record, diagnostics) = prospective_resource(context, at, request.directory)?; + return Ok(match record { + Some(record) => Resolved::Record(resource, record, diagnostics), + None => Resolved::Done(resource, None, diagnostics), + }); + } + Ok(match existing_target(context, request.target.as_ref())? { + ExistingTarget::Resource(resource, position, diagnostics) => { + Resolved::Record(resource, context.resource(position).clone(), diagnostics) + } + ExistingTarget::Missing(resource, diagnostic) => { + Resolved::Done(resource, None, vec![diagnostic]) + } + ExistingTarget::Declaration(resource) => { + let target: DeclarationTarget = match &request.target { + Some(QueryTarget::Identity(QualifiedIdentity::Declaration(identity))) => { + DeclarationTarget::Identity(identity.clone()) + } + _ => DeclarationTarget::Text(resource.target.clone()), + }; + let (explanation, diagnostics) = explain_declaration(context, &target, None)?; + Resolved::Done(resource, Some(Box::new(explanation)), diagnostics) + } + }) +} + +/// The index diagnostics, then the ones this query raised. +pub(crate) fn report_diagnostics( + context: &Context<'_>, + raised: Vec, +) -> Vec { + (0..context.source.lookup.diagnostics.len()) + .map(ReportDiagnostic::Index) + .chain(raised.into_iter().map(ReportDiagnostic::Raised)) + .collect() +} + +/// The report of a target with no record: its resource, explanation and diagnostics only. +pub(crate) fn unresolved_report( + context: &Context<'_>, + filters: &ReportFilters, + resolved: ( + ResourceReport, + Option>, + Vec, + ), +) -> ScopeReport { + let (resource, explanation, raised) = resolved; + let complete: bool = explanation.as_ref().is_some_and(|item| item.complete); + ScopeReport { + resource, + scope_chain: Vec::new(), + available: Vec::new(), + used: Vec::new(), + relationship_scope: Vec::new(), + nearby_unavailable: Vec::new(), + filters: filters.clone(), + sections: Vec::new(), + explanation: explanation.map(|explanation| *explanation), + move_preview: None, + diagnostics: report_diagnostics(context, raised), + complete, + } +} + +fn used_by<'a>(context: &Context<'a>, record: &ResourceRecord) -> Used<'a> { + let consumer: QualifiedIdentity = QualifiedIdentity::Resource(record.identity.clone()); + let mut used: Used<'a> = Used { + all: HashSet::new(), + direct: HashSet::new(), + related: HashSet::new(), + }; + for usage in context.usages_of_consumer(&consumer) { + let usage = context.usage(*usage); + used.all.insert(&usage.declaration); + if usage.through.is_none() { + used.direct.insert(&usage.declaration); + } else { + used.related.insert(&usage.declaration); + } + } + used +} + +/// Visible declarations the resource uses, keyed by identity, later ones replacing earlier ones. +fn used_records( + context: &Context<'_>, + direct: &Selected, + relationships: &Selected, + used: &Used<'_>, +) -> Selected { + let mut order: Vec<&DeclarationIdentity> = Vec::new(); + let mut keyed: HashMap<&DeclarationIdentity, (usize, Provenance)> = HashMap::new(); + let sources = [(direct, &used.direct), (relationships, &used.related)]; + for (values, wanted) in sources { + for (position, provenance) in values { + let identity: &DeclarationIdentity = &context.declaration(*position).identity; + if wanted.contains(identity) + && keyed + .insert(identity, (*position, provenance.clone())) + .is_none() + { + order.push(identity); + } + } + } + let mut records: Selected = order + .into_iter() + .filter_map(|identity| keyed.remove(identity)) + .collect(); + records.sort_by_cached_key(|(position, _)| report_key(context, *position)); + records +} + +fn provenance(reason: &str, through: Option) -> Provenance { + Provenance { + reason: reason.to_owned(), + through, + } +} + +/// The classification as `(position, provenance)` and `(position, reason)` lists. +fn classified( + classification: ReportClassification, +) -> (Selected, Selected, Selected) { + let direct: Selected = classification + .direct + .into_iter() + .map(|(position, reason)| (position, provenance(reason.as_str(), None))) + .collect(); + let relationships: Selected = classification + .relationships + .into_iter() + .map(|entry| { + ( + entry.position, + provenance(entry.reason.as_str(), Some(entry.through)), + ) + }) + .collect(); + let unavailable: Selected = classification + .unavailable + .into_iter() + .map(|(position, reason)| (position, reason.as_str().to_owned())) + .collect(); + (direct, relationships, unavailable) +} + +/// The non-global and used declarations, and how many globals were dropped. +fn without_unused_globals( + context: &Context<'_>, + values: Selected, + used: &HashSet<&DeclarationIdentity>, +) -> (Selected, i64) { + let before: usize = values.len(); + let kept: Selected = values + .into_iter() + .filter(|(position, _)| { + let record = context.declaration(*position); + record.scope != ScopeKind::Global || used.contains(&record.identity) + }) + .collect(); + let dropped: i64 = count(before - kept.len()); + (kept, dropped) +} + +/// Every section of `record`, filtered, trimmed to the requested sections and global policy. +fn sections( + context: &Context<'_>, + record: &ResourceRecord, + filters: &ReportFilters, +) -> Result { + let classification: ReportClassification = + classify_report_resource(context.source.records, context.source.lookup, record) + .map_err(ReportFailure::Path)?; + let (direct, relationships, unavailable) = classified(classification); + let used: Used<'_> = used_by(context, record); + let used_values: Selected = used_records(context, &direct, &relationships, &used); + let nearby_values: Selected = + nearby(context, record, unavailable, filters.nearby_depth).map_err(ReportFailure::Path)?; + let includes = |section: &str| filters.sections.iter().any(|name| name == section); + let mut direct: Selected = filtered(context, direct, filters, &used.all); + let mut used_values: Selected = filtered(context, used_values, filters, &used.all); + let mut relationships: Selected = + filtered(context, relationships, filters, &used.all); + let nearby_values: Selected = filtered(context, nearby_values, filters, &used.all); + if !includes(AVAILABLE_SECTION) { + direct.clear(); + } + if !includes(USED_SECTION) { + used_values.clear(); + } + if !includes(RELATIONSHIP_SECTION) { + relationships.clear(); + } + if filters.dependency_depth > 0 { + used_values = expand_dependencies(context, used_values, filters.dependency_depth); + } + let mut collapsed: i64 = 0; + if filters.globals == GLOBAL_SUMMARY_POLICY || filters.globals == GLOBAL_USED_POLICY { + let (kept, dropped) = without_unused_globals(context, direct, &used.all); + direct = kept; + if filters.globals == GLOBAL_SUMMARY_POLICY { + collapsed = dropped; + } + } + Ok(Sections { + direct, + used: used_values, + relationships, + nearby: nearby_values, + collapsed, + }) +} + +/// `len(nearby[:page_size])` with Python's slice semantics for a negative size. +fn nearby_shown(length: usize, page_size: i64) -> usize { + match usize::try_from(page_size) { + Ok(size) => size.min(length), + Err(_) => { + let dropped: usize = usize::try_from(page_size.unsigned_abs()).unwrap_or(usize::MAX); + length.saturating_sub(dropped) + } + } +} + +/// The nearby page and its section. +fn nearby_page( + context: &Context<'_>, + nearby: &Selected, + filters: &ReportFilters, +) -> Result<(Vec, ScopeSection), ReportFailure> { + let shown: usize = nearby_shown(nearby.len(), filters.page_size); + if shown == 0 && !nearby.is_empty() { + return Err(ReportFailure::IndexOutOfRange); + } + let reports: Vec = nearby[..shown] + .iter() + .map(|(position, reason)| { + declaration_report(context, *position, None, Some(reason.clone())) + }) + .collect(); + let truncated: bool = reports.len() < nearby.len(); + let section: ScopeSection = ScopeSection { + name: NEARBY_SECTION.to_owned(), + total: count(nearby.len()), + returned: count(reports.len()), + collapsed: false, + collapsed_count: 0, + truncated, + complete: context.completeness().static_visibility, + next_cursor: reports + .last() + .filter(|_| truncated) + .map(|report| report.identity.clone()), + cursor: None, + page_size: filters.page_size, + }; + Ok((reports, section)) +} + +/// The report of a resolved record: its pages, sections, scope chain and completeness. +pub(crate) fn record_report( + context: &Context<'_>, + filters: &ReportFilters, + resolved: (ResourceReport, ResourceRecord, Vec), +) -> Result { + let (resource, record, mut raised) = resolved; + let selected: Sections = sections(context, &record, filters)?; + let (cursor, cursor_diagnostic) = resolve_cursor( + context, + filters.cursor.as_deref(), + &[&selected.direct, &selected.used, &selected.relationships], + ); + raised.extend(cursor_diagnostic); + let completeness = context.completeness(); + let spec = |name: &'static str, complete: bool, collapsed: i64| PageSpec { + name, + cursor: cursor.as_deref(), + page_size: filters.page_size, + complete, + collapsed, + }; + let (available, available_section) = page( + context, + &selected.direct, + &spec( + AVAILABLE_SECTION, + completeness.static_visibility, + selected.collapsed, + ), + ); + let (used, used_section) = page( + context, + &selected.used, + &spec(USED_SECTION, completeness.runtime_usage, 0), + ); + let (relationship_scope, relationship_section) = page( + context, + &selected.relationships, + &spec(RELATIONSHIP_SECTION, completeness.relationships, 0), + ); + let (nearby_reports, nearby_section) = nearby_page(context, &selected.nearby, filters)?; + let sections: Vec = vec![ + available_section, + used_section, + relationship_section, + nearby_section, + ]; + let complete: bool = sections.iter().all(|section| section.complete) && !resource.prospective; + if resource.prospective { + raised.push(diagnostic( + INCOMPLETE_USAGE, + PROSPECTIVE_USAGE_MESSAGE.to_owned(), + resource.path.clone(), + )); + } + Ok(ScopeReport { + scope_chain: scope_chain(context, &record), + resource, + available, + used, + relationship_scope, + nearby_unavailable: if filters.include_nearby { + nearby_reports + } else { + Vec::new() + }, + filters: filters.clone(), + sections, + explanation: None, + move_preview: None, + diagnostics: report_diagnostics(context, raised), + complete, + }) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/selection.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/selection.rs new file mode 100644 index 0000000000..4f705f3038 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/selection.rs @@ -0,0 +1,372 @@ +//! Report section selection: filters, dependencies, nearby declarations, chains and pages. + +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, QualifiedIdentity, RejectedPath, ResourceRecord, + ScopeKind, +}; +use crate::scope_report::_helpers::context::Context; +use crate::scope_report::_helpers::declarations::{declaration_report, declaration_text}; +use crate::scope_report::_helpers::paths::{ancestry, normalized, part_count}; +use crate::scope_report::_helpers::targets::raised; +use crate::scope_report::constants::{ + CURRENT_PATH, DEPENDENCY_REASON, GLOBAL_CHAIN, IDENTITY_SEPARATOR, INHERITED_CHAIN, + INVALID_CURSOR, LOCAL_CHAIN, +}; +use crate::scope_report::models::{ + DeclarationReport, DiagnosticFacts, Provenance, ReportFilters, ScopeChainEntry, ScopeSection, +}; +use sqlbuild_core::posix_path::main::posix_parent::posix_parent; +use sqlbuild_core::text::main::python_fnmatchcase::python_fnmatchcase; +use std::collections::{BTreeSet, HashMap, HashSet}; + +/// Declarations at input positions with what each carries: a provenance or a hidden reason. +pub(crate) type Selected = Vec<(usize, T)>; + +/// `_filter_visible` and `_filter_nearby`: folder, kind, glob and used filters in order. +pub(crate) fn filtered( + context: &Context<'_>, + values: Selected, + filters: &ReportFilters, + used: &HashSet<&DeclarationIdentity>, +) -> Selected { + let mut selected: Selected = values; + if let Some(defined_under) = &filters.defined_under { + selected = defined_beneath(context, selected, defined_under); + } + if !filters.kinds.is_empty() { + selected.retain(|(position, _)| { + let kind: &str = context.declaration(*position).identity.kind.as_str(); + filters + .kinds + .iter() + .any(|selected_kind| selected_kind == kind) + }); + } + if let Some(glob) = &filters.glob { + selected.retain(|(position, _)| glob_matches(context.declaration(*position), glob)); + } + if filters.used_only { + selected.retain(|(position, _)| used.contains(&context.declaration(*position).identity)); + } + selected +} + +/// The items defined beneath `defined_under`; any path `normalize_path` rejects empties them. +fn defined_beneath( + context: &Context<'_>, + values: Selected, + defined_under: &str, +) -> Selected { + let Ok(under) = normalized(defined_under) else { + return Vec::new(); + }; + let mut kept: Selected = Vec::new(); + for item in values { + match ancestry(&context.declaration(item.0).path, &under) { + Ok(true) => kept.push(item), + Ok(false) => {} + Err(_) => return Vec::new(), + } + } + kept +} + +/// Whether the glob matches the declaration's name or qualified identity. +pub(crate) fn glob_matches(record: &DeclarationRecord, glob: &str) -> bool { + python_fnmatchcase(&record.identity.name, glob) + || python_fnmatchcase(&declaration_text(record), glob) +} + +/// `(format_identity, path)`: the order reports list declarations in. +pub(crate) fn report_key(context: &Context<'_>, position: usize) -> (String, String) { + let record: &DeclarationRecord = context.declaration(position); + (declaration_text(record), record.path.clone()) +} + +/// `_expand_dependencies`: the used declarations with `depth` levels of their dependencies. +pub(crate) fn expand_dependencies( + context: &Context<'_>, + values: Selected, + depth: i64, +) -> Selected { + let mut order: Vec<&DeclarationIdentity> = Vec::new(); + let mut selected: HashMap<&DeclarationIdentity, (usize, Provenance)> = HashMap::new(); + for (position, provenance) in values { + let identity: &DeclarationIdentity = &context.declaration(position).identity; + if selected.insert(identity, (position, provenance)).is_none() { + order.push(identity); + } + } + let mut frontier: Vec<&DeclarationIdentity> = order.clone(); + for _level in 0..depth.max(0) { + let mut dependencies: Vec<&DeclarationIdentity> = Vec::new(); + for identity in &frontier { + for position in context.declarations_of(identity) { + if let Some(macro_values) = context + .values(*position) + .and_then(|values| values.macro_values.as_ref()) + { + dependencies.extend(macro_values.dependencies.iter()); + } + } + let consumer: QualifiedIdentity = QualifiedIdentity::Declaration((*identity).clone()); + dependencies.extend( + context + .usages_of_consumer(&consumer) + .iter() + .map(|usage| &context.usage(*usage).declaration), + ); + } + let mut next: Vec<&DeclarationIdentity> = Vec::new(); + for identity in dependencies { + if !selected.contains_key(identity) && !next.contains(&identity) { + next.push(identity); + } + } + for identity in &next { + for position in context.declarations_of(identity) { + let provenance: Provenance = Provenance { + reason: DEPENDENCY_REASON.to_owned(), + through: None, + }; + if selected.insert(identity, (*position, provenance)).is_none() { + order.push(identity); + } + } + } + frontier = next; + } + let mut expanded: Selected = order + .into_iter() + .filter_map(|identity| selected.remove(identity)) + .collect(); + expanded.sort_by_cached_key(|(position, _)| report_key(context, *position)); + expanded +} + +/// `_nearby`: hidden declarations in the resource's own root, folder line or relationships. +pub(crate) fn nearby( + context: &Context<'_>, + resource: &ResourceRecord, + unavailable: Selected, + depth: i64, +) -> Result, RejectedPath> { + let parent: String = posix_parent(&resource.path); + let parent_parent: String = posix_parent(&parent); + let depth: i64 = depth.max(0); + let parent_parts: i64 = i64::try_from(part_count(&parent)).unwrap_or(i64::MAX); + let related: HashSet<&DeclarationIdentity> = context + .grants_of_resource(&resource.identity) + .iter() + .map(|grant| &context.grant(*grant).declaration) + .collect(); + let mut result: Selected = Vec::new(); + for (position, reason) in unavailable { + let record: &DeclarationRecord = context.declaration(position); + if record.scope == ScopeKind::Private + && record.identity.owner.as_ref() != Some(&resource.identity) + { + continue; + } + let owner: &str = record + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(&record.ownership_root.path); + let same_root: bool = record.ownership_root.path == resource.ownership_root.path; + let ancestor: bool = ancestry(&parent, owner)?; + let connected: bool = related.contains(&record.identity); + let sibling: bool = depth >= 1 && posix_parent(owner) == parent_parent; + let descendant: bool = ancestry(owner, &parent)? + && i64::try_from(part_count(owner)).unwrap_or(i64::MAX) - parent_parts <= depth; + if connected || (same_root && (ancestor || sibling || descendant)) { + result.push((position, reason)); + } + } + Ok(result) +} + +/// `_scope_chain`: the resource's folder, its ancestors to the root, then the global scope. +pub(crate) fn scope_chain( + context: &Context<'_>, + resource: &ResourceRecord, +) -> Vec { + let parent: String = posix_parent(&resource.path); + let root: &str = &resource.ownership_root.path; + let mut paths: Vec = vec![parent.clone()]; + let mut current: String = parent.clone(); + while current != CURRENT_PATH && current != root { + let next: String = posix_parent(¤t); + if next == current { + break; + } + current = next; + paths.push(current.clone()); + } + paths.push(root.to_owned()); + let mut unique: Vec = Vec::new(); + for path in paths { + if !unique.contains(&path) { + unique.push(path); + } + } + unique.sort_by(|left, right| { + part_count(right) + .cmp(&part_count(left)) + .then_with(|| left.cmp(right)) + }); + let mut counts: HashMap<&str, i64> = HashMap::new(); + let mut global: i64 = 0; + for position in context.canonical_declarations() { + let record: &DeclarationRecord = context.declaration(*position); + if record.scope == ScopeKind::Global { + global += 1; + continue; + } + let owner: &str = record + .owning_path + .as_deref() + .filter(|path| !path.is_empty()) + .unwrap_or(&record.ownership_root.path); + *counts.entry(owner).or_default() += 1; + } + let mut entries: Vec = unique + .into_iter() + .map(|path| ScopeChainEntry { + kind: if path == parent { + LOCAL_CHAIN + } else { + INHERITED_CHAIN + } + .to_owned(), + declaration_count: counts.get(path.as_str()).copied().unwrap_or(0), + path, + }) + .collect(); + entries.push(ScopeChainEntry { + kind: GLOBAL_CHAIN.to_owned(), + path: GLOBAL_CHAIN.to_owned(), + declaration_count: global, + }); + entries +} + +/// A section's name, page size, completeness and collapsed global count. +pub(crate) struct PageSpec<'a> { + pub(crate) name: &'a str, + pub(crate) cursor: Option<&'a str>, + pub(crate) page_size: i64, + pub(crate) complete: bool, + pub(crate) collapsed: i64, +} + +/// `len(values)` as Python counts it. +pub(crate) fn count(length: usize) -> i64 { + i64::try_from(length).unwrap_or(i64::MAX) +} + +/// The text after the last `:`, as `identity.rsplit(":", 1)[-1]` reads it. +fn bare_name(identity: &str) -> &str { + identity + .rsplit_once(IDENTITY_SEPARATOR) + .map_or(identity, |(_, name)| name) +} + +/// The most times `identity` appears in any one section. +fn most_in_one_section(sections: &[Vec], identity: &str) -> usize { + sections + .iter() + .map(|section| occurrences(section, identity)) + .max() + .unwrap_or(0) +} + +/// How many times `identity` appears in one section. +fn occurrences(section: &[String], identity: &str) -> usize { + section.iter().filter(|item| *item == identity).count() +} + +/// The qualified identity text of each declaration in one section. +fn section_texts(context: &Context<'_>, section: &Selected) -> Vec { + section + .iter() + .map(|(position, _)| declaration_text(context.declaration(*position))) + .collect() +} + +/// `_resolve_cursor`: a qualified or uniquely bare identity appearing once per section. +pub(crate) fn resolve_cursor( + context: &Context<'_>, + requested: Option<&str>, + sections: &[&Selected], +) -> (Option, Option) { + let Some(requested) = requested else { + return (None, None); + }; + let identities: Vec> = sections + .iter() + .map(|section| section_texts(context, section)) + .collect(); + let unique: BTreeSet<&String> = identities.iter().flatten().collect(); + let qualified: bool = requested.contains(IDENTITY_SEPARATOR); + if qualified + && unique.iter().any(|identity| identity.as_str() == requested) + && most_in_one_section(&identities, requested) == 1 + { + return (Some(requested.to_owned()), None); + } + let bare: Vec<&&String> = unique + .iter() + .filter(|identity| bare_name(identity) == requested) + .collect(); + if let [resolved] = bare.as_slice() + && !qualified + && most_in_one_section(&identities, resolved) == 1 + { + return (Some((**resolved).clone()), None); + } + (None, Some(raised(INVALID_CURSOR, "Cursor", requested))) +} + +/// `_page`: the reports after the cursor, one page long, and the section describing them. +pub(crate) fn page( + context: &Context<'_>, + values: &Selected, + spec: &PageSpec<'_>, +) -> (Vec, ScopeSection) { + let size: usize = usize::try_from(spec.page_size.max(0)).unwrap_or(usize::MAX); + let start: usize = spec + .cursor + .and_then(|cursor| { + values.iter().position(|(position, _)| { + declaration_text(context.declaration(*position)) == cursor + }) + }) + .map_or(0, |index| index + 1); + let end: usize = start.saturating_add(size).min(values.len()); + let selected: &[(usize, Provenance)] = values.get(start..end).unwrap_or_default(); + let reports: Vec = selected + .iter() + .map(|(position, provenance)| { + declaration_report(context, *position, Some(provenance.clone()), None) + }) + .collect(); + let truncated: bool = start + selected.len() < values.len(); + let next_cursor: Option = reports + .last() + .filter(|_| truncated) + .map(|report| report.identity.clone()); + let section: ScopeSection = ScopeSection { + name: spec.name.to_owned(), + total: count(values.len()) + spec.collapsed, + returned: count(reports.len()), + collapsed: spec.collapsed > 0, + collapsed_count: spec.collapsed, + truncated, + complete: spec.complete, + next_cursor, + cursor: spec.cursor.map(str::to_owned), + page_size: count(size), + }; + (reports, section) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/targets.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/targets.rs new file mode 100644 index 0000000000..899463d91b --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/targets.rs @@ -0,0 +1,366 @@ +//! Report targets: existing resources, prospective paths, declaration explanations and errors. + +use crate::scope_index::main::explain_report_declaration::explain_report_declaration; +use crate::scope_index::main::format_identity::format_identity; +use crate::scope_index::main::identity_repr::identity_repr; +use crate::scope_index::main::parse_identity::parse_identity; +use crate::scope_index::main::report_target::report_target; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationKind, DeclarationStanding, OwnershipRoot, QualifiedIdentity, + ReportTarget, ResourceIdentity, ResourceKind, ResourceRecord, TargetFailure, TargetMatch, +}; +use crate::scope_report::_helpers::context::Context; +use crate::scope_report::_helpers::declarations::declaration_report; +use crate::scope_report::_helpers::paths::{ + ancestry, default_suffix, looks_like_path, normalized, part_count, resource_path_is_valid, +}; +use crate::scope_report::constants::{ + DIRECT_CHILD_STEM, DUPLICATE_DECLARATION, ERROR_SEVERITY, IDENTITY_SEPARATOR, + INVALID_PROSPECTIVE_PATH, UNKNOWN_TARGET, UNQUALIFIED_TARGET, +}; +use crate::scope_report::models::{ + DeclarationTarget, DiagnosticFacts, Explanation, Provenance, QueryTarget, ReportFailure, + ResourceReport, +}; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +/// A diagnostic the query raises, with only a code, message and optional path. +pub(crate) fn diagnostic(code: &str, message: String, path: Option) -> DiagnosticFacts { + DiagnosticFacts { + code: code.to_owned(), + message, + severity: ERROR_SEVERITY.to_owned(), + path, + line: None, + column: None, + declaration: None, + resource: None, + } +} + +/// `_diagnostic`: `{label} {value!r} is not valid for this scope query`. +pub(crate) fn raised(code: &str, label: &str, value: &str) -> DiagnosticFacts { + let shown: String = python_str_repr(active_python_text(), value); + diagnostic( + code, + format!("{label} {shown} is not valid for this scope query"), + None, + ) +} + +/// `{value!r}` of Python text. +pub(crate) fn shown(value: &str) -> String { + python_str_repr(active_python_text(), value) +} + +/// `str(target)`: identity targets print their dataclass `repr`. +pub(crate) fn target_text(target: &QueryTarget) -> String { + match target { + QueryTarget::Identity(identity) => identity_repr(identity), + QueryTarget::Text(text) | QueryTarget::Path(text) => text.clone(), + } +} + +/// The raw target text a report shows: qualified identity text, or the text itself. +pub(crate) fn raw_target(target: &QueryTarget) -> String { + match target { + QueryTarget::Identity(identity) => format_identity(identity), + QueryTarget::Text(text) | QueryTarget::Path(text) => text.clone(), + } +} + +/// Python's `query_target`: matching resource or declaration positions, or why it raises. +pub(crate) fn query( + context: &Context<'_>, + target: &QueryTarget, +) -> Result { + let (target, text): (ReportTarget, String) = match target { + QueryTarget::Identity(identity) => { + (ReportTarget::Identity(identity.clone()), String::new()) + } + QueryTarget::Text(text) | QueryTarget::Path(text) => { + (ReportTarget::Text(text.clone()), text.clone()) + } + }; + report_target(context.source.records, context.source.lookup, &target).map_err(|failure| { + match failure { + TargetFailure::Identity(failure) => ReportFailure::Identity(failure, text), + TargetFailure::Path(rejected) => ReportFailure::Path(rejected), + } + }) +} + +/// The resource positions a query matched; declaration matches name no resource. +pub(crate) fn resource_matches(matched: TargetMatch) -> Vec { + match matched { + TargetMatch::Resources(positions) => positions, + TargetMatch::Declarations(_) => Vec::new(), + } +} + +/// `_unknown_target_diagnostic`. +pub(crate) fn unknown_target(value: &str) -> DiagnosticFacts { + let target_type: &str = if looks_like_path(value) { + "path" + } else { + "qualified identity" + }; + diagnostic( + crate::scope_report::constants::UNKNOWN_TARGET, + format!("Unknown {target_type} target {}", shown(value)), + None, + ) +} + +/// Whether text names a declaration kind before its first `:`. +pub(crate) fn names_declaration(text: &str) -> bool { + let kind: &str = text + .split_once(IDENTITY_SEPARATOR) + .map_or(text, |(kind, _)| kind); + DeclarationKind::parse(kind).is_some() +} + +/// `_ownership_roots`: index and resource roots, deepest first, then by path and kind. +pub(crate) fn ownership_roots(context: &Context<'_>) -> Vec { + let mut roots: Vec = Vec::new(); + let candidates = context.canonical_roots().chain( + context + .canonical_resources() + .map(|record| &record.ownership_root), + ); + for root in candidates { + if !roots.contains(root) { + roots.push(root.clone()); + } + } + roots.sort_by_cached_key(|root| { + ( + std::cmp::Reverse(part_count(&root.path)), + root.path.clone(), + root.resource_kind + .map(ResourceKind::as_str) + .unwrap_or_default(), + ) + }); + roots +} + +/// `_prospective_resource`: the report and stand-in record of a path that may not exist yet. +pub(crate) fn prospective_resource( + context: &Context<'_>, + at: &str, + directory: bool, +) -> Result<(ResourceReport, Option, Vec), ReportFailure> { + let report = |path: Option| ResourceReport { + target: at.to_owned(), + identity: None, + path, + prospective: true, + directory, + duplicate_count: 0, + }; + let invalid = || raised(INVALID_PROSPECTIVE_PATH, "Target", at); + let Ok(path) = normalized(at) else { + return Ok((report(None), None, vec![invalid()])); + }; + let mut found: Option = None; + for root in ownership_roots(context) { + if ancestry(&path, &root.path).map_err(ReportFailure::Path)? { + found = Some(root); + break; + } + } + let Some(root) = found.filter(|root| directory || resource_path_is_valid(&path, root)) else { + return Ok((report(Some(path)), None, vec![invalid()])); + }; + let authored: String = if directory { + format!("{path}/{DIRECT_CHILD_STEM}{}", default_suffix(&root)) + } else { + path.clone() + }; + let record: ResourceRecord = ResourceRecord { + identity: ResourceIdentity { + kind: root.resource_kind.unwrap_or(ResourceKind::Model), + name: format!(""), + }, + path: authored, + ownership_root: root, + }; + Ok((report(Some(path)), Some(record), Vec::new())) +} + +/// What `_existing_target` resolved: the report, the resource, or a declaration to explain. +pub(crate) enum ExistingTarget { + Resource(ResourceReport, usize, Vec), + Declaration(ResourceReport), + Missing(ResourceReport, DiagnosticFacts), +} + +/// `_existing_target` without its declaration explanation, which the caller builds. +pub(crate) fn existing_target( + context: &Context<'_>, + target: Option<&QueryTarget>, +) -> Result { + let missing = |raw: &str, diagnostic: DiagnosticFacts| { + ExistingTarget::Missing( + ResourceReport { + target: raw.to_owned(), + ..ResourceReport::default() + }, + diagnostic, + ) + }; + let Some(target) = target else { + return Ok(missing( + "", + raised( + crate::scope_report::constants::UNQUALIFIED_TARGET, + "Target", + "", + ), + )); + }; + let raw: String = raw_target(target); + let declaration: bool = match target { + QueryTarget::Identity(identity) => matches!(identity, QualifiedIdentity::Declaration(_)), + QueryTarget::Text(text) => names_declaration(text), + QueryTarget::Path(_) => false, + }; + if declaration { + return Ok(ExistingTarget::Declaration(ResourceReport { + target: raw.clone(), + identity: Some(raw), + ..ResourceReport::default() + })); + } + if let QueryTarget::Text(text) = target + && !text.contains(IDENTITY_SEPARATOR) + && !looks_like_path(text) + { + return Ok(missing( + &raw, + raised( + crate::scope_report::constants::UNQUALIFIED_TARGET, + "Target", + &raw, + ), + )); + } + let matches: Vec = match query(context, target) { + Ok(TargetMatch::Declarations(positions)) if !positions.is_empty() => { + return Err(ReportFailure::IndexOutOfRange); + } + Ok(matched) => resource_matches(matched), + Err(ReportFailure::Path(_) | ReportFailure::Identity(..)) => { + return Ok(missing(&raw, unknown_target(&raw))); + } + Err(failure) => return Err(failure), + }; + let Some(first) = matches.first().copied() else { + return Ok(missing(&raw, unknown_target(&raw))); + }; + let mut diagnostics: Vec = Vec::new(); + if matches.len() > 1 { + diagnostics.push(diagnostic( + crate::scope_report::constants::DUPLICATE_RESOURCE, + format!("Target {} matches {} resources", shown(&raw), matches.len()), + None, + )); + } + let record: &ResourceRecord = context.resource(first); + Ok(ExistingTarget::Resource( + ResourceReport { + target: raw, + identity: Some(format_identity(&QualifiedIdentity::Resource( + record.identity.clone(), + ))), + path: Some(crate::scope_report::_helpers::paths::safe_scope_path( + &record.path, + )), + prospective: false, + directory: false, + duplicate_count: i64::try_from(matches.len()).unwrap_or(i64::MAX), + }, + first, + diagnostics, + )) +} + +/// The explanation of `target` at `at`, with its target diagnostics. +pub(crate) fn explain_declaration( + context: &Context<'_>, + target: &DeclarationTarget, + at: Option<&ResourceRecord>, +) -> Result<(Explanation, Vec), ReportFailure> { + let unexplained = Explanation { + declaration: None, + complete: false, + }; + let identity: Option = match target { + DeclarationTarget::Identity(identity) => Some(identity.clone()), + DeclarationTarget::Text(text) => match parse_identity(text) { + Ok(QualifiedIdentity::Declaration(identity)) => Some(identity), + _ => None, + }, + }; + let Some(identity) = identity else { + let text: String = match target { + DeclarationTarget::Text(text) => text.clone(), + DeclarationTarget::Identity(identity) => { + format_identity(&QualifiedIdentity::Declaration(identity.clone())) + } + }; + return Ok(( + unexplained, + vec![raised(UNQUALIFIED_TARGET, "Target", &text)], + )); + }; + let qualified: String = format_identity(&QualifiedIdentity::Declaration(identity.clone())); + let records: &[usize] = context.declarations_of(&identity); + let Some(first) = records.first().copied() else { + return Ok(( + unexplained, + vec![raised(UNKNOWN_TARGET, "Target", &qualified)], + )); + }; + let mut diagnostics: Vec = Vec::new(); + if records.len() > 1 { + diagnostics.push(raised(DUPLICATE_DECLARATION, "Target", &qualified)); + } + let mut visibility: Option = None; + let mut inaccessible: Option = None; + if let Some(resource) = at { + let standing: DeclarationStanding = explain_report_declaration( + context.source.records, + context.source.lookup, + resource, + context.declaration(first), + ) + .map_err(ReportFailure::Path)?; + match standing { + DeclarationStanding::Visible(reason) => { + visibility = Some(Provenance { + reason: reason.as_str().to_owned(), + through: None, + }); + } + DeclarationStanding::Granted(reason, through) => { + visibility = Some(Provenance { + reason: reason.as_str().to_owned(), + through: Some(through), + }); + } + DeclarationStanding::Inaccessible(reason) => { + inaccessible = Some(reason.as_str().to_owned()); + } + } + } + Ok(( + Explanation { + declaration: Some(declaration_report(context, first, visibility, inaccessible)), + complete: context.completeness().placement, + }, + diagnostics, + )) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/constants.rs b/crates/sqlbuild-scopes/src/scope_report/constants.rs new file mode 100644 index 0000000000..12858b381d --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/constants.rs @@ -0,0 +1,39 @@ +//! Section names, policies, labels and messages of scope reports. + +pub(crate) const AVAILABLE_SECTION: &str = "available"; +pub(crate) const USED_SECTION: &str = "used"; +pub(crate) const RELATIONSHIP_SECTION: &str = "relationship_scope"; +pub(crate) const NEARBY_SECTION: &str = "nearby_unavailable"; +pub(crate) const LIST_SECTION: &str = "list"; +pub(crate) const GLOBAL_SUMMARY_POLICY: &str = "summary"; +pub(crate) const GLOBAL_USED_POLICY: &str = "used"; +pub(crate) const IDENTITY_SEPARATOR: char = ':'; +pub(crate) const PATH_SEPARATOR: char = '/'; +pub(crate) const WINDOWS_PATH_SEPARATOR: char = '\\'; +pub(crate) const CURRENT_PATH: &str = "."; +pub(crate) const GROUP_DIRECTORY: &str = "_sqlbuild"; +pub(crate) const INVALID_PATH_TEXT: &str = ""; +pub(crate) const DIRECT_CHILD_STEM: &str = ""; +pub(crate) const SQL_SUFFIX: &str = ".sql"; +pub(crate) const YML_SUFFIX: &str = ".yml"; +pub(crate) const YAML_SUFFIX: &str = ".yaml"; +pub(crate) const DEPENDENCY_REASON: &str = "dependency"; +pub(crate) const LOCAL_CHAIN: &str = "local"; +pub(crate) const INHERITED_CHAIN: &str = "inherited"; +pub(crate) const GLOBAL_CHAIN: &str = "global"; +pub(crate) const ERROR_SEVERITY: &str = "error"; +pub(crate) const ESCAPE_CHARACTER: char = '\u{1b}'; +pub(crate) const ENUM_MEMBER_PREVIEW: usize = 20; +pub(crate) const RESULT_SCHEMA_VERSION: i64 = 2; +pub(crate) const METADATA_SCHEMA_VERSION: i64 = 3; +pub(crate) const PROSPECTIVE_USAGE_MESSAGE: &str = + "Runtime usage and relationship facts are unavailable for a prospective path"; + +/// Python's `ScopeDiagnosticCode` values the reports raise. +pub(crate) const UNKNOWN_TARGET: &str = "S001"; +pub(crate) const UNQUALIFIED_TARGET: &str = "S002"; +pub(crate) const DUPLICATE_DECLARATION: &str = "S003"; +pub(crate) const INVALID_PROSPECTIVE_PATH: &str = "S011"; +pub(crate) const INCOMPLETE_USAGE: &str = "S013"; +pub(crate) const INVALID_CURSOR: &str = "S014"; +pub(crate) const DUPLICATE_RESOURCE: &str = "S018"; diff --git a/crates/sqlbuild-scopes/src/scope_report/main/browse_scope_folders.rs b/crates/sqlbuild-scopes/src/scope_report/main/browse_scope_folders.rs new file mode 100644 index 0000000000..b1e46e7cc4 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/browse_scope_folders.rs @@ -0,0 +1,10 @@ +//! `browse_scope_folders`: the direct child declaration-definition folders of one folder. + +use crate::scope_report::_helpers::folders::browsing; +use crate::scope_report::models::{BrowseResult, FolderRequest, ReportSource}; + +/// The browse result of `request` over one lookup. +#[must_use] +pub fn browse_scope_folders(source: ReportSource<'_>, request: &FolderRequest) -> BrowseResult { + browsing(source, request) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/build_scope_report.rs b/crates/sqlbuild-scopes/src/scope_report/main/build_scope_report.rs new file mode 100644 index 0000000000..a3bcf6fc5c --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/build_scope_report.rs @@ -0,0 +1,23 @@ +//! `query_scope_report`: what an existing target or prospective path can see, page by page. + +use crate::scope_report::_helpers::context::Context; +use crate::scope_report::_helpers::report::{Resolved, record_report, resolve, unresolved_report}; +use crate::scope_report::models::{ReportFailure, ReportRequest, ReportSource, ScopeReport}; + +/// The report of `request` over one lookup, or the path or identity error Python raises. +pub fn build_scope_report( + source: ReportSource<'_>, + request: &ReportRequest, +) -> Result { + let context: Context<'_> = Context::new(source); + match resolve(&context, request)? { + Resolved::Record(resource, record, raised) => { + record_report(&context, &request.filters, (resource, record, raised)) + } + Resolved::Done(resource, explanation, raised) => Ok(unresolved_report( + &context, + &request.filters, + (resource, explanation, raised), + )), + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/explain_scope_declaration.rs b/crates/sqlbuild-scopes/src/scope_report/main/explain_scope_declaration.rs new file mode 100644 index 0000000000..0d17410e4c --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/explain_scope_declaration.rs @@ -0,0 +1,33 @@ +//! `explain_scope_declaration`: a qualified declaration in an optional resource or path context. + +use crate::scope_index::models::ResourceRecord; +use crate::scope_report::_helpers::context::Context; +use crate::scope_report::_helpers::targets::explain_declaration; +use crate::scope_report::_helpers::targets::{prospective_resource, query, resource_matches}; +use crate::scope_report::models::{ + DiagnosticFacts, ExplainRequest, Explanation, ReportFailure, ReportSource, +}; + +/// The explanation and its diagnostics, or the path or identity error Python raises. +pub fn explain_scope_declaration( + source: ReportSource<'_>, + request: &ExplainRequest, +) -> Result<(Explanation, Vec), ReportFailure> { + let context: Context<'_> = Context::new(source); + let mut diagnostics: Vec = Vec::new(); + let mut at: Option = None; + if let Some(path) = &request.at { + let (_resource, record, raised) = prospective_resource(&context, path, request.directory)?; + at = record; + diagnostics = raised; + } else if let Some(target) = &request.target { + let matches: Vec = resource_matches(query(&context, target)?); + if let [position] = matches.as_slice() { + at = Some(context.resource(*position).clone()); + } + } + let (explanation, explained) = + explain_declaration(&context, &request.declaration, at.as_ref())?; + diagnostics.extend(explained); + Ok((explanation, diagnostics)) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/list_scope_declarations.rs b/crates/sqlbuild-scopes/src/scope_report/main/list_scope_declarations.rs new file mode 100644 index 0000000000..675a39ed76 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/list_scope_declarations.rs @@ -0,0 +1,12 @@ +//! `list_scope_declarations`: declarations recursively beneath one definition folder. + +use crate::scope_report::_helpers::folders::listing; +use crate::scope_report::models::{FolderRequest, ListResult, ReportFailure, ReportSource}; + +/// The list result of `request`, or the path error Python raises. +pub fn list_scope_declarations( + source: ReportSource<'_>, + request: &FolderRequest, +) -> Result { + listing(source, request) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/mod.rs b/crates/sqlbuild-scopes/src/scope_report/main/mod.rs new file mode 100644 index 0000000000..ed557b8a5e --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/mod.rs @@ -0,0 +1,7 @@ +pub mod browse_scope_folders; +pub mod build_scope_report; +pub mod explain_scope_declaration; +pub mod list_scope_declarations; +pub mod preview_scope_move; +pub mod scope_metadata_projection; +pub mod serialize_scope_result; diff --git a/crates/sqlbuild-scopes/src/scope_report/main/preview_scope_move.rs b/crates/sqlbuild-scopes/src/scope_report/main/preview_scope_move.rs new file mode 100644 index 0000000000..351ab76435 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/preview_scope_move.rs @@ -0,0 +1,14 @@ +//! `preview_scope_move`: how a resource's visible declarations change if it moves. + +use crate::scope_report::_helpers::moves::move_preview; +use crate::scope_report::models::{ + DiagnosticFacts, MovePreview, MoveRequest, ReportFailure, ReportSource, +}; + +/// The move preview of `request`, or why it cannot be previewed. +pub fn preview_scope_move( + source: ReportSource<'_>, + request: &MoveRequest, +) -> Result<(Option, Vec), ReportFailure> { + move_preview(source, request) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/scope_metadata_projection.rs b/crates/sqlbuild-scopes/src/scope_report/main/scope_metadata_projection.rs new file mode 100644 index 0000000000..117f83fb9e --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/scope_metadata_projection.rs @@ -0,0 +1,12 @@ +//! `scope_metadata_projection`: deterministic, value-free scope metadata of one index. + +use sqlbuild_core::json::models::JsonValue; + +use crate::scope_report::_helpers::projection::projection; +use crate::scope_report::models::ProjectionIndex; + +/// The projection Python's `build_projection` returns, as a JSON object. +#[must_use] +pub fn scope_metadata_projection(index: &ProjectionIndex) -> JsonValue { + projection(index) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/main/serialize_scope_result.rs b/crates/sqlbuild-scopes/src/scope_report/main/serialize_scope_result.rs new file mode 100644 index 0000000000..0982a2932f --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/main/serialize_scope_result.rs @@ -0,0 +1,60 @@ +//! `serialize_scope_report`: deterministic schema-version-two ASCII JSON of a query result. + +use sqlbuild_core::json::models::JsonValue; + +use crate::scope_report::_helpers::declarations::integer; +use crate::scope_report::_helpers::json::{ + canonical_json, chain_json, declarations_json, diagnostics_json, explanation_json, + filters_json, folder_json, move_json, object, resource_json, section_json, +}; +use crate::scope_report::constants::RESULT_SCHEMA_VERSION; +use crate::scope_report::models::{ReportFailure, ScopeResult}; + +/// The result's JSON text with a trailing newline. +pub fn serialize_scope_result(result: &ScopeResult) -> Result { + let schema: (&str, JsonValue) = ("schema_version", integer(RESULT_SCHEMA_VERSION)); + let payload: JsonValue = match result { + ScopeResult::Report(report) => object(vec![ + schema, + ("resource", resource_json(&report.resource)), + ("scope_chain", chain_json(&report.scope_chain)), + ("available", declarations_json(&report.available)), + ("used", declarations_json(&report.used)), + ( + "relationship_scope", + declarations_json(&report.relationship_scope), + ), + ( + "nearby_unavailable", + declarations_json(&report.nearby_unavailable), + ), + ("filters", filters_json(&report.filters)), + ( + "sections", + JsonValue::Array(report.sections.iter().map(section_json).collect()), + ), + ("explanation", explanation_json(report.explanation.as_ref())), + ("move_preview", move_json(report.move_preview.as_ref())), + ("diagnostics", diagnostics_json(&report.diagnostics)?), + ("complete", JsonValue::Bool(report.complete)), + ]), + ScopeResult::Browse(browse) => object(vec![ + schema, + ("folder", JsonValue::String(browse.folder.clone())), + ( + "folders", + JsonValue::Array(browse.folders.iter().map(folder_json).collect()), + ), + ("diagnostics", diagnostics_json(&browse.diagnostics)?), + ("complete", JsonValue::Bool(browse.complete)), + ]), + ScopeResult::List(list) => object(vec![ + schema, + ("folder", JsonValue::String(list.folder.clone())), + ("declarations", declarations_json(&list.declarations)), + ("section", section_json(&list.section)), + ("diagnostics", diagnostics_json(&list.diagnostics)?), + ]), + }; + Ok(canonical_json(&payload)? + "\n") +} diff --git a/crates/sqlbuild-scopes/src/scope_report/mod.rs b/crates/sqlbuild-scopes/src/scope_report/mod.rs new file mode 100644 index 0000000000..b8cee1a9ab --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/mod.rs @@ -0,0 +1,9 @@ +//! `sqb scope` reports: queries, folder browsing, move previews, projections and serialization. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-scopes/src/scope_report/models.rs b/crates/sqlbuild-scopes/src/scope_report/models.rs new file mode 100644 index 0000000000..ef5f54c765 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/models.rs @@ -0,0 +1,369 @@ +//! Report facts, requests and the report results Python's scope report models mirror. + +use sqlbuild_core::json::models::JsonValue; + +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationRecord, GrantKind, IdentityFailure, OwnershipRoot, + QualifiedIdentity, RejectedPath, ReportLookup, ReportRecords, ResourceIdentity, ResourceRecord, +}; + +/// Python's `MacroMetadata`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct MacroValues { + pub parameters: Vec, + pub dependencies: Vec, + pub source_digest: String, +} + +/// Python's `EnumMetadata`: member names and the scalar type. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct EnumValues { + pub members: Vec, + pub scalar_type: String, +} + +/// Python's `ConstantMetadata`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ConstantValues { + pub logical_type: String, + pub collection_kind: Option, + pub item_count: Option, + pub nullable: bool, + pub render_as: Option, +} + +/// One declaration's value metadata. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DeclarationValues { + pub macro_values: Option, + pub enum_values: Option, + pub constant_values: Option, +} + +/// Python's `ScopeCompleteness`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Completeness { + pub discovery: bool, + pub static_visibility: bool, + pub runtime_usage: bool, + pub relationships: bool, + pub placement: bool, + pub promotion_impact: bool, +} + +impl Default for Completeness { + fn default() -> Self { + Self { + discovery: true, + static_visibility: true, + runtime_usage: true, + relationships: true, + placement: true, + promotion_impact: true, + } + } +} + +/// What reports read beside the lookup: each record's value metadata, in input order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ReportValues { + pub declarations: Vec, + pub completeness: Completeness, +} + +/// A lookup's records and groups with the values reports read. +#[derive(Clone, Copy, Debug)] +pub struct ReportSource<'a> { + pub records: &'a ReportRecords, + pub lookup: &'a ReportLookup, + pub values: &'a ReportValues, +} + +/// What a query names: an identity, text naming an identity or path, or a `PurePath`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum QueryTarget { + Identity(QualifiedIdentity), + Text(String), + /// A `PurePath`, which reads as path text without the text-only identity checks. + Path(String), +} + +/// Python's `ScopeReportFilters`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReportFilters { + pub sections: Vec, + pub include_nearby: bool, + pub defined_under: Option, + /// Declaration kind values. + pub kinds: Vec, + pub glob: Option, + pub used_only: bool, + pub dependency_depth: i64, + pub sort: String, + pub cursor: Option, + pub page_size: i64, + pub nearby_depth: i64, + pub globals: String, +} + +/// `query_scope_report`'s arguments. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReportRequest { + pub target: Option, + pub at: Option, + pub directory: bool, + pub filters: ReportFilters, +} + +/// A declaration to explain: an identity, or text naming one. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DeclarationTarget { + Identity(DeclarationIdentity), + Text(String), +} + +/// `explain_scope_declaration`'s arguments. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExplainRequest { + pub declaration: DeclarationTarget, + pub target: Option, + pub at: Option, + pub directory: bool, +} + +/// `browse_scope_folders` and `list_scope_declarations` arguments. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FolderRequest { + pub folder: String, + pub target: Option, + pub prospective: bool, + /// The list filters; `None` lists everything with every global. + pub filters: Option, +} + +/// `preview_scope_move`'s arguments; `destination` is `str(destination)`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MoveRequest { + pub resource: QueryTarget, + pub destination: String, +} + +/// Python's `ScopeDiagnostic` as a report projects it: identities as qualified text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiagnosticFacts { + pub code: String, + pub message: String, + pub severity: String, + pub path: Option, + pub line: Option, + pub column: Option, + pub declaration: Option, + pub resource: Option, +} + +/// A report diagnostic: the index diagnostic at a canonical position, or one the query raised. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ReportDiagnostic { + Index(usize), + Raised(DiagnosticFacts), +} + +/// Python's `VisibilityProvenance`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Provenance { + pub reason: String, + pub through: Option, +} + +/// Python's `SourceLocation`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SourceLocation { + pub path: String, + pub line: i64, + pub column: i64, +} + +/// Python's `DeclarationReport`; metadata values are JSON-ready, arrays standing for tuples. +#[derive(Clone, Debug, PartialEq)] +pub struct DeclarationReport { + pub identity: String, + pub kind: String, + pub name: String, + pub owner: Option, + pub definition: SourceLocation, + pub scope: String, + pub owning_path: Option, + pub visibility: Option, + pub inaccessible_reason: Option, + pub metadata: Vec<(String, JsonValue)>, + pub consumers: Vec, + pub dependencies: Vec, + pub grants: Vec, + pub required_scope: Option, + pub required_path: Option, + pub promotion_impact: Vec, +} + +/// Python's `ScopeChainEntry`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopeChainEntry { + pub kind: String, + pub path: String, + pub declaration_count: i64, +} + +/// Python's `ScopeSection`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopeSection { + pub name: String, + pub total: i64, + pub returned: i64, + pub collapsed: bool, + pub collapsed_count: i64, + pub truncated: bool, + pub complete: bool, + pub next_cursor: Option, + pub cursor: Option, + pub page_size: i64, +} + +/// Python's `ScopeResourceReport`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResourceReport { + pub target: String, + pub identity: Option, + pub path: Option, + pub prospective: bool, + pub directory: bool, + pub duplicate_count: i64, +} + +/// Python's `DeclarationExplanation`. +#[derive(Clone, Debug, PartialEq)] +pub struct Explanation { + pub declaration: Option, + pub complete: bool, +} + +/// Python's `MovePreview`. +#[derive(Clone, Debug, PartialEq)] +pub struct MovePreview { + pub resource: String, + pub destination: String, + pub new_ownership_root: Option, + pub retained: Vec, + pub gained: Vec, + pub lost: Vec, + pub invalidated_usages: Vec, + pub private_retained: Vec, + pub relationship_retained: Vec, + pub complete: bool, +} + +/// Python's `ScopeFolder`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScopeFolder { + pub path: String, + pub name: String, + pub descendant_count: i64, + pub child_count: i64, + pub used_count: i64, + pub kind_counts: Vec<(String, i64)>, +} + +/// Python's `ScopeReport`. +#[derive(Clone, Debug, PartialEq)] +pub struct ScopeReport { + pub resource: ResourceReport, + pub scope_chain: Vec, + pub available: Vec, + pub used: Vec, + pub relationship_scope: Vec, + pub nearby_unavailable: Vec, + pub filters: ReportFilters, + pub sections: Vec, + pub explanation: Option, + pub move_preview: Option, + pub diagnostics: Vec, + pub complete: bool, +} + +/// Python's `ScopeBrowseResult`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BrowseResult { + pub folder: String, + pub folders: Vec, + pub diagnostics: Vec, + pub complete: bool, +} + +/// Python's `ScopeListResult`. +#[derive(Clone, Debug, PartialEq)] +pub struct ListResult { + pub folder: String, + pub declarations: Vec, + pub section: ScopeSection, + pub diagnostics: Vec, +} + +/// Any scope query result `serialize_scope_report` accepts. +#[derive(Clone, Debug, PartialEq)] +pub enum ScopeResult { + Report(Box), + Browse(BrowseResult), + List(ListResult), +} + +/// Why a query raises: a path `normalize_path` rejects, or an identity `parse_identity` rejects. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ReportFailure { + Path(RejectedPath), + /// The failure and the text Python parsed. + Identity(IdentityFailure, String), + /// Serialization met an index diagnostic it cannot project or a value JSON cannot hold. + Serialization(String), + /// An input Python's report code reads past an empty result for, raising `IndexError`. + IndexOutOfRange, +} + +/// One usage of a full scope index, with the enum member the metadata projection shows. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectionUsage { + pub consumer: QualifiedIdentity, + pub declaration: DeclarationIdentity, + pub kind: String, + pub through: Option, + pub enum_member: Option, +} + +/// One relationship grant of a full scope index. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectionGrant { + pub resource: ResourceIdentity, + pub declaration: DeclarationIdentity, + pub through: QualifiedIdentity, + pub kind: GrantKind, +} + +/// A stored visible or inaccessible fact: resource, declaration, reason and what it came through. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProjectionStanding { + pub resource: ResourceIdentity, + pub declaration: DeclarationIdentity, + pub reason: String, + pub through: Option, +} + +/// A full `ScopeIndex` in its tuple orders, for `scope_metadata_projection`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ProjectionIndex { + pub roots: Vec, + pub resources: Vec, + pub declarations: Vec, + pub values: ReportValues, + pub usages: Vec, + pub grants: Vec, + pub visibility: Vec, + pub inaccessible: Vec, + pub diagnostics: Vec, +} diff --git a/crates/sqlbuild-scopes/src/scope_report/tests.rs b/crates/sqlbuild-scopes/src/scope_report/tests.rs new file mode 100644 index 0000000000..8c53e82309 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_scope_report.rs"] +mod scope_report; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-scopes/src/scope_report/tests/helpers.rs b/crates/sqlbuild-scopes/src/scope_report/tests/helpers.rs new file mode 100644 index 0000000000..a39c88f97f --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/tests/helpers.rs @@ -0,0 +1,128 @@ +use crate::scope_index::main::report_lookup::report_lookup; +use crate::scope_index::models::{ + DeclarationIdentity, DeclarationKind, DeclarationRecord, OwnershipRoot, OwnershipRootKind, + QualifiedIdentity, ReportLookup, ReportRecords, ReportUsage, ResourceIdentity, ResourceKind, + ResourceRecord, ScopeKind, +}; +use crate::scope_report::models::{ + DeclarationValues, MacroValues, QueryTarget, ReportFilters, ReportValues, +}; + +/// One model under `models/sales`, a global macro, an inherited constant and a local enum. +pub(super) fn orders_facts() -> (ReportRecords, ReportLookup, ReportValues) { + let models: OwnershipRoot = OwnershipRoot { + path: "models".to_owned(), + kind: OwnershipRootKind::Resource, + resource_kind: Some(ResourceKind::Model), + }; + let project: OwnershipRoot = OwnershipRoot { + path: ".".to_owned(), + kind: OwnershipRootKind::Global, + resource_kind: None, + }; + let orders: ResourceIdentity = ResourceIdentity { + kind: ResourceKind::Model, + name: "orders".to_owned(), + }; + let declaration = + |kind, name: &str, path: &str, scope, root: &OwnershipRoot, owner: Option<&str>| { + DeclarationRecord { + identity: DeclarationIdentity { + kind, + name: name.to_owned(), + owner: None, + }, + path: path.to_owned(), + line: 1, + column: 1, + scope, + ownership_root: root.clone(), + owning_path: owner.map(str::to_owned), + dependencies: Vec::new(), + } + }; + let declarations: Vec = vec![ + declaration( + DeclarationKind::Macro, + "g", + "macros/g.py", + ScopeKind::Global, + &project, + None, + ), + declaration( + DeclarationKind::Constant, + "c", + "models/constants/c.sql", + ScopeKind::Inherited, + &models, + Some("models"), + ), + declaration( + DeclarationKind::Enum, + "e", + "models/finance/_enums/e.sql", + ScopeKind::Local, + &models, + Some("models/finance"), + ), + ]; + let records: ReportRecords = ReportRecords { + roots: vec![models.clone()], + resources: vec![ResourceRecord { + identity: orders.clone(), + path: "models/sales/orders.sql".to_owned(), + ownership_root: models, + }], + usages: vec![ReportUsage { + consumer: QualifiedIdentity::Resource(orders), + declaration: declarations[1].identity.clone(), + kind: "runtime".to_owned(), + through: None, + }], + declarations, + ..ReportRecords::default() + }; + let lookup: ReportLookup = report_lookup(&records).expect("valid paths"); + let values: ReportValues = ReportValues { + declarations: vec![ + DeclarationValues { + macro_values: Some(MacroValues { + parameters: vec!["x".to_owned()], + ..MacroValues::default() + }), + ..DeclarationValues::default() + }, + DeclarationValues::default(), + DeclarationValues::default(), + ], + ..ReportValues::default() + }; + (records, lookup, values) +} + +/// Every section, nearby declarations included, and every global listed. +pub(super) fn open_filters() -> ReportFilters { + ReportFilters { + sections: vec![ + "available".to_owned(), + "used".to_owned(), + "relationship_scope".to_owned(), + ], + include_nearby: true, + defined_under: None, + kinds: Vec::new(), + glob: None, + used_only: false, + dependency_depth: 0, + sort: "identity".to_owned(), + cursor: None, + page_size: 100, + nearby_depth: 1, + globals: "all".to_owned(), + } +} + +pub(super) fn text_target(target: Option<&str>) -> Option { + target.map(|text| QueryTarget::Text(text.to_owned())) +} diff --git a/crates/sqlbuild-scopes/src/scope_report/tests/test_scope_report.rs b/crates/sqlbuild-scopes/src/scope_report/tests/test_scope_report.rs new file mode 100644 index 0000000000..88102b678d --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/tests/test_scope_report.rs @@ -0,0 +1,180 @@ +use crate::scope_report::_helpers::paths::ancestry; +use crate::scope_report::main::browse_scope_folders::browse_scope_folders; +use crate::scope_report::main::build_scope_report::build_scope_report; +use crate::scope_report::main::serialize_scope_result::serialize_scope_result; +use crate::scope_report::models::{ + FolderRequest, ReportRequest, ReportSource, ScopeReport, ScopeResult, +}; +use crate::scope_report::tests::helpers::{open_filters, orders_facts, text_target}; +use crate::scope_report::tests::test_types::{ + DescendantTestCase, ReportTestCase, SerializeTestCase, +}; + +#[test] +fn given_target_or_prospective_path_when_reporting_then_sections_follow_scope_rules() { + let test_cases = [ + ReportTestCase { + description: "an existing model sees globals and inherited declarations", + target: Some("model:orders"), + at: None, + expected_available: &["constant:c", "macro:g"], + expected_used: &["constant:c"], + expected_nearby: &["enum:e"], + expected_chain: &[ + "local models/sales 0", + "inherited models 1", + "global global 1", + ], + expected_complete: true, + }, + ReportTestCase { + description: "a prospective path in the enum's folder sees it but stays incomplete", + target: None, + at: Some("models/finance/new.sql"), + expected_available: &["constant:c", "enum:e", "macro:g"], + expected_used: &[], + expected_nearby: &[], + expected_chain: &[ + "local models/finance 1", + "inherited models 1", + "global global 1", + ], + expected_complete: false, + }, + ]; + for test_case in test_cases { + let (records, lookup, values) = orders_facts(); + let source: ReportSource<'_> = ReportSource { + records: &records, + lookup: &lookup, + values: &values, + }; + let request: ReportRequest = ReportRequest { + target: text_target(test_case.target), + at: test_case.at.map(str::to_owned), + directory: false, + filters: open_filters(), + }; + + let report: ScopeReport = build_scope_report(source, &request).expect("valid paths"); + + let identities = |reports: &[crate::scope_report::models::DeclarationReport]| { + reports + .iter() + .map(|report| report.identity.clone()) + .collect::>() + }; + assert_eq!( + ( + identities(&report.available), + identities(&report.used), + identities(&report.nearby_unavailable), + report + .scope_chain + .iter() + .map(|entry| format!( + "{} {} {}", + entry.kind, entry.path, entry.declaration_count + )) + .collect::>(), + report.complete, + ), + ( + test_case + .expected_available + .iter() + .map(|item| (*item).to_owned()) + .collect(), + test_case + .expected_used + .iter() + .map(|item| (*item).to_owned()) + .collect(), + test_case + .expected_nearby + .iter() + .map(|item| (*item).to_owned()) + .collect(), + test_case + .expected_chain + .iter() + .map(|item| (*item).to_owned()) + .collect(), + test_case.expected_complete, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_browse_result_when_serializing_then_json_is_sorted_compact_ascii() { + let test_cases = [SerializeTestCase { + description: "the project root lists the global and model declaration folders", + folder: ".", + expected_json: concat!( + "{\"complete\":true,\"diagnostics\":[],\"folder\":\".\",\"folders\":[", + "{\"child_count\":1,\"descendant_count\":1,\"kind_counts\":{\"macro\":1},", + "\"name\":\"global\",\"path\":\"global\",\"used_count\":0},", + "{\"child_count\":2,\"descendant_count\":2,\"kind_counts\":{\"constant\":1,\"enum\":1},", + "\"name\":\"models\",\"path\":\"models\",\"used_count\":1}],", + "\"schema_version\":2}\n" + ), + }]; + for test_case in test_cases { + let (records, lookup, values) = orders_facts(); + let source: ReportSource<'_> = ReportSource { + records: &records, + lookup: &lookup, + values: &values, + }; + let request: FolderRequest = FolderRequest { + folder: test_case.folder.to_owned(), + target: None, + prospective: false, + filters: None, + }; + + let result = ScopeResult::Browse(browse_scope_folders(source, &request)); + + assert_eq!( + serialize_scope_result(&result).expect("serializes"), + test_case.expected_json, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_paths_when_comparing_ancestry_then_whole_components_decide() { + let test_cases = [ + DescendantTestCase { + description: "a shared name prefix is not an ancestor", + path: "models/orders", + ancestor: "models/order", + expected_descendant: false, + }, + DescendantTestCase { + description: "the project root contains every path", + path: "models/orders", + ancestor: ".", + expected_descendant: true, + }, + DescendantTestCase { + description: "paths compare after normalization", + path: "models\\orders\\..\\sales", + ancestor: "models/./sales", + expected_descendant: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + ancestry(test_case.path, test_case.ancestor).expect("valid paths"), + test_case.expected_descendant, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-scopes/src/scope_report/tests/test_types.rs b/crates/sqlbuild-scopes/src/scope_report/tests/test_types.rs new file mode 100644 index 0000000000..75c622ef83 --- /dev/null +++ b/crates/sqlbuild-scopes/src/scope_report/tests/test_types.rs @@ -0,0 +1,24 @@ +pub(super) struct ReportTestCase { + pub(super) description: &'static str, + pub(super) target: Option<&'static str>, + pub(super) at: Option<&'static str>, + pub(super) expected_available: &'static [&'static str], + pub(super) expected_used: &'static [&'static str], + pub(super) expected_nearby: &'static [&'static str], + /// `kind path count` of each scope chain entry. + pub(super) expected_chain: &'static [&'static str], + pub(super) expected_complete: bool, +} + +pub(super) struct DescendantTestCase { + pub(super) description: &'static str, + pub(super) path: &'static str, + pub(super) ancestor: &'static str, + pub(super) expected_descendant: bool, +} + +pub(super) struct SerializeTestCase { + pub(super) description: &'static str, + pub(super) folder: &'static str, + pub(super) expected_json: &'static str, +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs index 15fa52285e..a4dfc9aaf1 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/mod.rs @@ -1,3 +1,4 @@ pub mod declaration_references; pub mod model_headers; +pub mod select_items; pub mod sql_interpolation; diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/columns.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/columns.rs new file mode 100644 index 0000000000..fd6098a717 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/columns.rs @@ -0,0 +1,104 @@ +//! Locate the `columns (...)` body of a MODEL header and each column entry in it. + +use crate::compiler::_helpers::model_headers::tokenization::{ + HeaderToken, SYMBOL_TOKEN, WORD_TOKEN, +}; +use crate::compiler::models::{HeaderColumnSpan, HeaderColumns}; +use sqlbuild_core::constants::{CLOSE_PAREN, OPEN_PAREN}; + +const COLUMNS_KEY: &str = "columns"; + +fn is_symbol(token: &HeaderToken, symbol: &str) -> bool { + token.0 == SYMBOL_TOKEN && token.1 == symbol +} + +pub(crate) fn columns_open(tokens: &[HeaderToken]) -> Option { + let mut depth: i64 = 0; + for (index, token) in tokens.iter().enumerate() { + let next_opens = tokens + .get(index + 1) + .is_some_and(|next| next.1 == OPEN_PAREN); + if token.0 == WORD_TOKEN && token.1 == COLUMNS_KEY && depth == 0 && next_opens { + return Some(index + 1); + } + if is_symbol(token, OPEN_PAREN) { + depth += 1; + } else if is_symbol(token, CLOSE_PAREN) { + depth -= 1; + } + } + None +} + +/// The index of the token closing metadata opened at `open`, matched by value as Python does. +fn metadata_close(tokens: &[HeaderToken], open: usize) -> Option { + let mut depth: i64 = 1; + for (index, candidate) in tokens.iter().enumerate().skip(open + 1) { + if candidate.1 == OPEN_PAREN { + depth += 1; + } else if candidate.1 == CLOSE_PAREN { + depth -= 1; + if depth == 0 { + return Some(index); + } + } + } + None +} + +fn record_entry( + entries: Vec<(String, HeaderColumnSpan)>, + name: &str, + span: HeaderColumnSpan, +) -> Vec<(String, HeaderColumnSpan)> { + let mut updated = entries; + match updated.iter_mut().find(|(existing, _)| existing == name) { + Some(entry) => entry.1 = span, + None => updated.push((name.to_owned(), span)), + } + updated +} + +/// Scan the tokens of a header starting at code point `header_start` from its columns opener. +pub(crate) fn scan_columns( + tokens: &[HeaderToken], + open: usize, + header_start: usize, +) -> Result, String> { + let mut entries: Vec<(String, HeaderColumnSpan)> = Vec::new(); + let mut depth: i64 = 1; + let mut index = open + 1; + while index < tokens.len() { + let token = &tokens[index]; + if is_symbol(token, OPEN_PAREN) { + depth += 1; + } else if is_symbol(token, CLOSE_PAREN) { + depth -= 1; + if depth == 0 { + return Ok(Some(HeaderColumns { + body_start: header_start + tokens[open].2 + 1, + body_end: header_start + token.2, + entries, + })); + } + } else if token.0 == WORD_TOKEN && depth == 1 { + let metadata_open = tokens + .get(index + 1) + .ok_or_else(|| "tuple index out of range".to_owned())?; + if metadata_open.1 == OPEN_PAREN + && let Some(close) = metadata_close(tokens, index + 1) + { + let span = HeaderColumnSpan { + entry_start: header_start + token.2, + entry_end: header_start + tokens[close].2 + 1, + metadata_start: header_start + metadata_open.2 + 1, + metadata_end: header_start + tokens[close].2, + }; + entries = record_entry(entries, &token.1, span); + index = close; + } + } + index += 1; + } + Ok(None) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/locations.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/locations.rs new file mode 100644 index 0000000000..b96828bc1a --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/locations.rs @@ -0,0 +1,58 @@ +//! Authored locations of the column names a MODEL header declares. + +use crate::compiler::_helpers::model_headers::tokenization::parse_batch; +use crate::compiler::_helpers::select_items::scanning::{insert_named, line_starts}; +use crate::compiler::models::TextLocation; + +const NO_LOCATIONS: &str = "Native MODEL header parser returned no column locations"; + +/// Python's `header_column_locations` for `header` authored at code point `header_start`. +pub(crate) fn header_column_locations( + contents: &str, + header: &str, + header_start: usize, +) -> Result, String> { + let (_, offsets, error) = parse_batch(&[header.to_owned()])?.pop().unwrap_or(( + None, + None, + Some(NO_LOCATIONS.to_owned()), + )); + if let Some(error) = error { + return Err(error); + } + let offsets = offsets.ok_or_else(|| NO_LOCATIONS.to_owned())?; + let characters: Vec = contents.chars().collect(); + let starts = line_starts(&characters[..header_start.min(characters.len())]); + let header_line = starts.len(); + let header_column = header_start - starts[starts.len() - 1] + 1; + let header_characters: Vec = header.chars().collect(); + let (mut line, mut column, mut cursor) = (1, 1, 0); + let mut locations: Vec<(String, TextLocation)> = Vec::new(); + for (name, position, length) in offsets { + while cursor < position { + match header_characters[cursor..position] + .iter() + .position(|character| *character == '\n') + { + Some(newline) => { + line += 1; + column = 1; + cursor += newline + 1; + } + None => { + column += position - cursor; + cursor = position; + } + } + } + let start_column = column + if line == 1 { header_column - 1 } else { 0 }; + let location = TextLocation { + line: header_line + line - 1, + column: start_column, + end_line: header_line + line - 1, + end_column: start_column + length, + }; + locations = insert_named(locations, name, location); + } + Ok(locations) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/mod.rs index 8ca7357664..ad087e81da 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/mod.rs @@ -1,2 +1,5 @@ +pub mod columns; +pub mod locations; pub mod matching; pub mod tokenization; +pub mod words; diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/words.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/words.rs new file mode 100644 index 0000000000..1830816d01 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/model_headers/words.rs @@ -0,0 +1,55 @@ +//! Bare MODEL header words read as booleans, null, integers or floats. + +use crate::compiler::models::HeaderWord; +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::python_decimal_value::python_decimal_value; + +pub(crate) const TRUE_WORD: &str = "true"; +pub(crate) const FALSE_WORD: &str = "false"; +pub(crate) const NULL_WORD: &str = "null"; +const DECIMAL_POINT: char = '.'; +const NEGATIVE_SIGN: &str = "-"; +const ZERO_DIGITS: &str = "0"; + +/// The ASCII form of a run of Python decimal digits, or `None` if any character is not one. +fn ascii_digits(text: &str) -> Option { + let python = active_python_text(); + text.chars() + .map(|character| { + python_decimal_value(python, character).and_then(|value| char::from_digit(value, 10)) + }) + .collect() +} + +pub(crate) fn signed(word: &str) -> (&str, &str) { + match word.strip_prefix(['+', '-']) { + Some(rest) => (&word[..1], rest), + None => ("", word), + } +} + +pub(crate) fn integer(sign: &str, body: &str) -> Option { + let digits = ascii_digits(body).filter(|digits| !digits.is_empty())?; + let trimmed = digits.trim_start_matches('0'); + let magnitude = if trimmed.is_empty() { "0" } else { trimmed }; + let negative = sign == NEGATIVE_SIGN && magnitude != ZERO_DIGITS; + Some(HeaderWord::Integer(if negative { + format!("-{magnitude}") + } else { + magnitude.to_owned() + })) +} + +pub(crate) fn float(sign: &str, body: &str) -> Option { + let (whole, fraction) = body.split_once(DECIMAL_POINT)?; + let whole = ascii_digits(whole)?; + let fraction = ascii_digits(fraction)?; + if whole.is_empty() && fraction.is_empty() { + return None; + } + Some( + format!("{sign}{whole}.{fraction}0") + .parse::() + .map_or(HeaderWord::Text, HeaderWord::Float), + ) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/select_items/mod.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/select_items/mod.rs new file mode 100644 index 0000000000..0b5df88d2a --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/select_items/mod.rs @@ -0,0 +1 @@ +pub mod scanning; diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/select_items/scanning.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/select_items/scanning.rs new file mode 100644 index 0000000000..fd67392da7 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/select_items/scanning.rs @@ -0,0 +1,364 @@ +//! The top-level SELECT list of a model's SQL and the output name of each simple item. + +use sqlbuild_core::text::main::active_python_text::active_python_text; +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; +use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::main::python_upper::python_upper; + +const SELECT_KEYWORD: &str = "SELECT"; +const UNION_KEYWORD: &str = "UNION"; +const FROM_KEYWORD: &str = "FROM"; +const UNION_LOWER: &str = "union"; +const ESCAPE: char = '\\'; +const QUOTES: [char; 2] = ['\'', '"']; +const SCAN_SPECIAL: [char; 9] = ['\'', '"', '`', '(', ')', 's', 'S', 'f', 'F']; +const UNION_INITIALS: [char; 2] = ['u', 'U']; +const IDENTIFIER_SEPARATOR: char = '_'; +const DOUBLE_QUOTE: char = '"'; +const CLOSE_PAREN: char = ')'; +/// Letters `[A-Za-z]` matches under `re.IGNORECASE` besides the ASCII ones. +const IGNORECASE_EXTRA_LETTERS: [char; 4] = ['\u{130}', '\u{131}', '\u{17f}', '\u{212a}']; +const IGNORECASE_S: [char; 3] = ['s', 'S', '\u{17f}']; +const AS_LENGTH: usize = 2; +/// `end_column` is one past the last character, which is itself one-based. +const END_COLUMN_OFFSET: usize = 2; + +fn is_special(character: char) -> bool { + SCAN_SPECIAL.contains(&character) || UNION_INITIALS.contains(&character) +} + +/// `_keyword_at`: `keyword` at `index`, not inside a longer word. +fn keyword_at(sql: &[char], keyword: &str, index: usize) -> bool { + let end = index + keyword.chars().count(); + let slice: String = sql[index..end.min(sql.len())].iter().collect(); + if python_upper(active_python_text(), &slice) != keyword { + return false; + } + let word = |character: Option<&char>| { + character.is_some_and(|character| { + is_python_alnum(active_python_text(), *character) || *character == IDENTIFIER_SEPARATOR + }) + }; + let before = index.checked_sub(1).and_then(|position| sql.get(position)); + !word(before) && !word(sql.get(end)) +} + +fn has_union_candidate(sql: &[char]) -> bool { + let lowered: String = sql.iter().collect::().to_lowercase(); + lowered.contains(UNION_LOWER) +} + +/// The scan state of `_top_level_select_list_bounds`. +#[derive(Clone, Copy)] +struct ListScan { + depth: usize, + start: Option, + end: Option, +} + +/// What one character does to the scan: advance by a count, or finish with a result. +enum Step { + Advance(usize), + Jump(usize), + Finish(Option<(usize, usize)>), +} + +fn top_level_step(sql: &[char], index: usize, scan: ListScan, unions: bool) -> (Step, ListScan) { + let mut scan = scan; + let character = sql[index]; + if character == '(' { + scan.depth += 1; + return (Step::Advance(1), scan); + } + if character == CLOSE_PAREN { + scan.depth = scan.depth.saturating_sub(1); + return (Step::Advance(1), scan); + } + if scan.depth != 0 { + return (Step::Advance(1), scan); + } + let upper = character.to_ascii_uppercase(); + let Some(start) = scan.start else { + if upper == 'S' && keyword_at(sql, SELECT_KEYWORD, index) { + let start = index + SELECT_KEYWORD.len(); + scan.start = Some(start); + return (Step::Jump(start), scan); + } + return (Step::Advance(1), scan); + }; + if upper == 'U' && keyword_at(sql, UNION_KEYWORD, index) { + return (Step::Finish(None), scan); + } + if scan.end.is_none() && upper == 'F' && keyword_at(sql, FROM_KEYWORD, index) { + scan.end = Some(index); + if !unions { + return (Step::Finish(Some((start, index))), scan); + } + } + (Step::Advance(1), scan) +} + +/// `_top_level_select_list_bounds`: the SELECT list of the first top-level SELECT. +pub(crate) fn select_list_bounds(sql: &[char]) -> Option<(usize, usize)> { + let unions = has_union_candidate(sql); + let mut scan = ListScan { + depth: 0, + start: None, + end: None, + }; + let mut quote: Option = None; + let mut index = 0; + while index < sql.len() { + if let Some(open) = quote { + if sql[index] == ESCAPE { + index += 2; + continue; + } + if sql[index] == open { + quote = None; + } + index += 1; + continue; + } + let Some(offset) = sql[index..] + .iter() + .position(|character| is_special(*character)) + else { + break; + }; + index += offset; + if QUOTES.contains(&sql[index]) { + quote = Some(sql[index]); + index += 1; + continue; + } + let (step, next) = top_level_step(sql, index, scan, unions); + scan = next; + match step { + Step::Advance(count) => index += count, + Step::Jump(position) => index = position, + Step::Finish(bounds) => return bounds, + } + } + scan.start + .map(|start| (start, scan.end.unwrap_or(sql.len()))) +} + +/// `_split_top_level_select_items`: item ranges split at top-level commas. +pub(crate) fn split_select_items(sql: &[char], start: usize, end: usize) -> Vec<(usize, usize)> { + let mut items: Vec<(usize, usize)> = Vec::new(); + let mut depth: usize = 0; + let mut item_start = start; + let mut quote: Option = None; + let mut index = start; + while index < end { + let character = sql[index]; + if let Some(open) = quote { + if character == ESCAPE { + index += 2; + continue; + } + if character == open { + quote = None; + } + index += 1; + continue; + } + if QUOTES.contains(&character) { + quote = Some(character); + } else if character == '(' { + depth += 1; + } else if character == CLOSE_PAREN { + depth = depth.saturating_sub(1); + } else if character == ',' && depth == 0 { + items.push((item_start, index)); + item_start = index + 1; + } + index += 1; + } + items.push((item_start, end)); + items +} + +fn is_identifier_start(character: char, ignore_case: bool) -> bool { + character.is_ascii_alphabetic() + || character == IDENTIFIER_SEPARATOR + || (ignore_case && IGNORECASE_EXTRA_LETTERS.contains(&character)) +} + +fn is_identifier_part(character: char, ignore_case: bool) -> bool { + is_identifier_start(character, ignore_case) || character.is_ascii_digit() +} + +fn trim_end(text: &[char]) -> &[char] { + let end = text + .iter() + .rposition(|character| !is_python_space(*character)) + .map_or(0, |position| position + 1); + &text[..end] +} + +fn trim_start(text: &[char]) -> &[char] { + let start = text + .iter() + .position(|character| !is_python_space(*character)) + .unwrap_or(text.len()); + &text[start..] +} + +/// The start of an `IDENT|"[^"]+"` name ending `text`, or `None`. +fn trailing_name_start(text: &[char], ignore_case: bool) -> Option { + if text.last() == Some(&DOUBLE_QUOTE) { + let open = text[..text.len() - 1] + .iter() + .rposition(|character| *character == DOUBLE_QUOTE)?; + return (open + 2 < text.len()).then_some(open); + } + let start = text + .iter() + .rposition(|character| !is_identifier_part(*character, ignore_case)) + .map_or(0, |position| position + 1); + (start < text.len() && is_identifier_start(text[start], ignore_case)).then_some(start) +} + +fn is_full_name(text: &[char]) -> bool { + match text { + [DOUBLE_QUOTE, inner @ .., DOUBLE_QUOTE] => { + !inner.is_empty() && !inner.contains(&DOUBLE_QUOTE) + } + [first, rest @ ..] => { + is_identifier_start(*first, false) + && rest + .iter() + .all(|character| is_identifier_part(*character, false)) + } + [] => false, + } +} + +/// `\s+AS\s+NAME\s*\Z` under `re.IGNORECASE`. +fn explicit_alias(text: &[char]) -> Option<&[char]> { + let start = trailing_name_start(text, true)?; + let before = &text[..start]; + let spaced = trim_end(before); + if spaced.len() == before.len() || spaced.len() < AS_LENGTH { + return None; + } + let keyword = &spaced[spaced.len() - AS_LENGTH..]; + let is_as = matches!(keyword[0], 'a' | 'A') && IGNORECASE_S.contains(&keyword[1]); + let preceding = &spaced[..spaced.len() - AS_LENGTH]; + (is_as + && preceding + .last() + .is_some_and(|character| is_python_space(*character))) + .then_some(&text[start..]) +} + +/// `\s*(?:(?:IDENT|"[^"]+")\.)?NAME\s*\Z` from the start. +fn bare_reference(text: &[char]) -> Option<&[char]> { + let trimmed = trim_start(text); + if let Some(dot) = qualifier_end(trimmed) + && is_full_name(&trimmed[dot + 1..]) + { + return Some(&trimmed[dot + 1..]); + } + is_full_name(trimmed).then_some(trimmed) +} + +/// The index of the `.` after a leading `IDENT` or `"[^"]+"` qualifier. +fn qualifier_end(text: &[char]) -> Option { + let end = if text.first() == Some(&DOUBLE_QUOTE) { + let close = text[1..] + .iter() + .position(|character| *character == DOUBLE_QUOTE)? + + 1; + (close > 1).then_some(close + 1)? + } else { + let length = text + .iter() + .position(|character| !is_identifier_part(*character, false)) + .unwrap_or(text.len()); + (length > 0 && is_identifier_start(text[0], false)).then_some(length)? + }; + (text.get(end) == Some(&'.')).then_some(end) +} + +/// `\)\s+NAME\s*\Z`. +fn implicit_alias(text: &[char]) -> Option<&[char]> { + let start = trailing_name_start(text, false)?; + let before = &text[..start]; + let spaced = trim_end(before); + (spaced.len() < before.len() && spaced.last() == Some(&CLOSE_PAREN)).then_some(&text[start..]) +} + +/// `_select_item_output_name`: the output name of a simple SELECT item, quotes stripped. +pub(crate) fn select_item_output_name(item: &[char], implicit_aliases: bool) -> Option { + let text = trim_end(item); + let name = explicit_alias(text) + .or_else(|| bare_reference_with_trailing(item)) + .or_else(|| implicit_aliases.then(|| implicit_alias(text)).flatten())?; + let stripped: String = name.iter().collect(); + Some(stripped.trim_matches(DOUBLE_QUOTE).to_owned()) +} + +fn bare_reference_with_trailing(item: &[char]) -> Option<&[char]> { + bare_reference(trim_end(item)) +} + +/// `entries[name] = value`: a repeated name keeps its first position and takes the new value. +pub(crate) fn insert_named( + entries: Vec<(String, T)>, + name: String, + value: T, +) -> Vec<(String, T)> { + let mut updated = entries; + match updated.iter_mut().find(|(existing, _)| *existing == name) { + Some(entry) => entry.1 = value, + None => updated.push((name, value)), + } + updated +} + +/// The code-point offset where each line of `text` starts. +pub(crate) fn line_starts(text: &[char]) -> Vec { + std::iter::once(0) + .chain( + text.iter() + .enumerate() + .filter(|(_, character)| **character == '\n') + .map(|(index, _)| index + 1), + ) + .collect() +} + +/// `_location_for_absolute_span`: one-based lines and columns of `[start, end)`. +pub(crate) fn span_location( + starts: &[usize], + start: usize, + end: usize, +) -> crate::compiler::models::TextLocation { + let end = end.max(start); + let end_position = start.max(end.saturating_sub(1)); + let line_index = starts.partition_point(|line_start| *line_start <= start) - 1; + let end_line_index = starts.partition_point(|line_start| *line_start <= end_position) - 1; + crate::compiler::models::TextLocation { + line: line_index + 1, + column: start - starts[line_index] + 1, + end_line: end_line_index + 1, + end_column: end_position - starts[end_line_index] + END_COLUMN_OFFSET, + } +} + +/// The trimmed code-point span of a SELECT item, or `None` when it is blank. +pub(crate) fn trimmed_item(sql: &[char], (start, end): (usize, usize)) -> Option<(usize, usize)> { + let item = &sql[start..end]; + let leading = item + .iter() + .position(|character| !is_python_space(*character)) + .unwrap_or(item.len()); + let trailing = item + .iter() + .rposition(|character| !is_python_space(*character)) + .map_or(0, |position| position + 1); + (leading < trailing).then_some((start + leading, start + trailing)) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/header_word_value.rs b/crates/sqlbuild-sqltext/src/compiler/main/header_word_value.rs new file mode 100644 index 0000000000..3c44240450 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/header_word_value.rs @@ -0,0 +1,20 @@ +//! Read a bare MODEL header word as `true`, `false`, `null`, a number or text. + +use crate::compiler::_helpers::model_headers::words::{ + FALSE_WORD, NULL_WORD, TRUE_WORD, float, integer, signed, +}; +use crate::compiler::models::HeaderWord; + +/// Python's `_parse_word_value`: booleans, null, `^[+-]?\d+$` integers, decimal floats, or text. +pub fn header_word_value(word: &str) -> HeaderWord { + match word { + TRUE_WORD => return HeaderWord::Bool(true), + FALSE_WORD => return HeaderWord::Bool(false), + NULL_WORD => return HeaderWord::Null, + _ => {} + } + let (sign, body) = signed(word.strip_suffix('\n').unwrap_or(word)); + integer(sign, body) + .or_else(|| float(sign, body)) + .unwrap_or(HeaderWord::Text) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs index 7b066a290e..967e7982a8 100644 --- a/crates/sqlbuild-sqltext/src/compiler/main/mod.rs +++ b/crates/sqlbuild-sqltext/src/compiler/main/mod.rs @@ -1,10 +1,15 @@ pub mod declaration_references; +pub mod header_word_value; pub mod model_header_byte_matching; +pub mod model_header_column_locations; +pub mod model_header_columns; +pub mod model_header_entry_span; pub mod model_header_errors; pub mod model_header_matching; pub mod model_header_parsing; pub mod model_header_single_parsing; pub mod model_header_tokenizing; +pub mod model_output_columns; pub mod placeholder_defaults; pub mod sql_interpolation; pub mod statement_header_matching; diff --git a/crates/sqlbuild-sqltext/src/compiler/main/model_header_column_locations.rs b/crates/sqlbuild-sqltext/src/compiler/main/model_header_column_locations.rs new file mode 100644 index 0000000000..2fdfa2d484 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/model_header_column_locations.rs @@ -0,0 +1,16 @@ +//! Authored locations of the column names a MODEL header declares. + +use crate::compiler::models::TextLocation; + +/// Python's `header_column_locations` for `header` authored at code point `header_start`. +pub fn model_header_column_locations( + contents: &str, + header: &str, + header_start: usize, +) -> Result, String> { + crate::compiler::_helpers::model_headers::locations::header_column_locations( + contents, + header, + header_start, + ) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/model_header_columns.rs b/crates/sqlbuild-sqltext/src/compiler/main/model_header_columns.rs new file mode 100644 index 0000000000..7abfc94e24 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/model_header_columns.rs @@ -0,0 +1,23 @@ +//! The `columns (...)` body of a MODEL header and the span of each column entry. + +use crate::compiler::_helpers::model_headers::columns::{columns_open, scan_columns}; +use crate::compiler::_helpers::model_headers::matching::match_one; +use crate::compiler::_helpers::model_headers::tokenization::{HeaderToken, tokenize_one}; +use crate::compiler::models::HeaderColumns; + +/// Python's `model_header_columns_span`: code-point offsets of the columns body and entries. +pub fn model_header_columns(contents: &str) -> Result, String> { + let Some((header_start, header_end, _)) = match_one(contents) else { + return Ok(None); + }; + let header: String = contents + .chars() + .skip(header_start) + .take(header_end - header_start) + .collect(); + let tokens: Vec = tokenize_one(&header)?; + match columns_open(&tokens) { + Some(open) => scan_columns(&tokens, open, header_start), + None => Ok(None), + } +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/model_header_entry_span.rs b/crates/sqlbuild-sqltext/src/compiler/main/model_header_entry_span.rs new file mode 100644 index 0000000000..bc160580d9 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/model_header_entry_span.rs @@ -0,0 +1,38 @@ +//! Where one top-level `key value` entry of a MODEL header is authored. + +use crate::compiler::_helpers::model_headers::matching::match_one; +use crate::compiler::_helpers::model_headers::tokenization::{ + HeaderToken, SYMBOL_TOKEN, WORD_TOKEN, tokenize_one, +}; +use sqlbuild_core::constants::{CLOSE_PAREN, OPEN_PAREN}; +use sqlbuild_core::text::main::is_python_space::is_python_space; + +/// Code-point offsets of the entry from its key to its value's end, or the tokenizer's error. +pub fn model_header_entry_span( + contents: &str, + key: &str, +) -> Result, String> { + let Some((header_start, header_end, _)) = match_one(contents) else { + return Ok(None); + }; + let characters: Vec = contents.chars().collect(); + let header: String = characters[header_start..header_end].iter().collect(); + let tokens: Vec = tokenize_one(&header)?; + let mut depth: i64 = 0; + for (index, (kind, value, position)) in tokens.iter().enumerate() { + let symbol: bool = *kind == SYMBOL_TOKEN; + if symbol && value == OPEN_PAREN { + depth += 1; + } else if symbol && value == CLOSE_PAREN { + depth -= 1; + } else if depth == 0 && *kind == WORD_TOKEN && value == key && index + 2 < tokens.len() { + let start: usize = header_start + position; + let mut end: usize = header_start + tokens[index + 2].2; + while end > start && is_python_space(characters[end - 1]) { + end -= 1; + } + return Ok(Some((start, end))); + } + } + Ok(None) +} diff --git a/crates/sqlbuild-sqltext/src/compiler/main/model_output_columns.rs b/crates/sqlbuild-sqltext/src/compiler/main/model_output_columns.rs new file mode 100644 index 0000000000..fb84e8cf33 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/main/model_output_columns.rs @@ -0,0 +1,32 @@ +//! Authored locations of a model's simple top-level SELECT output columns. + +use crate::compiler::_helpers::model_headers::matching::match_one; +use crate::compiler::_helpers::select_items::scanning::{ + insert_named, line_starts, select_item_output_name, select_list_bounds, span_location, + split_select_items, trimmed_item, +}; +use crate::compiler::models::TextLocation; + +/// Python's `model_output_column_locations`: each simple output name and where it is authored. +pub fn model_output_columns(contents: &str, implicit_aliases: bool) -> Vec<(String, TextLocation)> { + let Some((_, _, sql_start)) = match_one(contents) else { + return Vec::new(); + }; + let characters: Vec = contents.chars().collect(); + let sql = &characters[sql_start..]; + let Some((start, end)) = select_list_bounds(sql) else { + return Vec::new(); + }; + let starts = line_starts(&characters); + let mut locations: Vec<(String, TextLocation)> = Vec::new(); + for range in split_select_items(sql, start, end) { + let Some(name) = select_item_output_name(&sql[range.0..range.1], implicit_aliases) else { + continue; + }; + if let Some((item_start, item_end)) = trimmed_item(sql, range) { + let location = span_location(&starts, sql_start + item_start, sql_start + item_end); + locations = insert_named(locations, name, location); + } + } + locations +} diff --git a/crates/sqlbuild-sqltext/src/compiler/models.rs b/crates/sqlbuild-sqltext/src/compiler/models.rs index 0ebf1118fd..46aa3f51a8 100644 --- a/crates/sqlbuild-sqltext/src/compiler/models.rs +++ b/crates/sqlbuild-sqltext/src/compiler/models.rs @@ -14,6 +14,18 @@ pub enum AuthoredValue { PythonHook(String, Vec<(String, AuthoredValue)>), } +/// A bare header word read as Python's header projection reads it. +#[derive(Clone, Debug, PartialEq)] +pub enum HeaderWord { + Bool(bool), + Null, + /// A signed integer in ASCII decimal digits. + Integer(String), + Float(f64), + /// Any other word, kept as text. + Text, +} + /// Whether a declaration reference names an enum member or a constant. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DeclarationReferenceKind { @@ -60,6 +72,32 @@ pub struct DeclarationReferenceScan { pub stop: Option, } +/// A one-based authored location: start line and column, and the column just past the end. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TextLocation { + pub line: usize, + pub column: usize, + pub end_line: usize, + pub end_column: usize, +} + +/// One `columns (...)` entry of a MODEL header: code-point offsets of the entry and its metadata. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HeaderColumnSpan { + pub entry_start: usize, + pub entry_end: usize, + pub metadata_start: usize, + pub metadata_end: usize, +} + +/// The `columns (...)` body of a MODEL header and each named entry in it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HeaderColumns { + pub body_start: usize, + pub body_end: usize, + pub entries: Vec<(String, HeaderColumnSpan)>, +} + /// A header whose values nest deeper than the parser allows, located in its file. #[derive(Clone, Debug, PartialEq, Eq)] pub struct NestingFailure { diff --git a/crates/sqlbuild-sqltext/src/compiler/tests.rs b/crates/sqlbuild-sqltext/src/compiler/tests.rs index 7744d6b316..c98ba89828 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests.rs @@ -2,6 +2,10 @@ mod declaration_references; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_model_columns.rs"] +mod model_columns; +#[path = "tests/test_model_header_entry_span.rs"] +mod model_header_entry_span; #[path = "tests/test_model_header_matching.rs"] mod model_header_matching; #[path = "tests/test_model_header_nesting.rs"] diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index bc850546cf..a4ac31e384 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -140,9 +140,18 @@ pub(crate) fn batch_sizes_bound_workers_by_contract() -> bool { /// Spell a scan as `kind:name[.member]@start..end, ...` plus ` | stop:` when it stops. pub(crate) fn scanned_references(sql: &str) -> Option { + spelled_scan(sql, false) +} + +/// The same spelling of the usage walk, which steps past malformed references. +pub(crate) fn scanned_usages(sql: &str) -> Option { + spelled_scan(sql, true) +} + +fn spelled_scan(sql: &str, skip_malformed: bool) -> Option { let python = python_text((3, 12), "15.0.0").expect("Python 3.12 is supported"); let scan: DeclarationReferenceScan = - scan_declaration_references(python, &[sql.to_owned()], false).pop()?; + scan_declaration_references(python, &[sql.to_owned()], skip_malformed).pop()?; let references: String = scan .references .iter() @@ -252,3 +261,23 @@ pub(crate) fn interpolation_facts( |result| (result.spans, result.reads), ) } + +/// Named locations from `[line, column, end_line, end_column]` rows. +pub(crate) fn locations( + columns: Vec<(&str, [usize; 4])>, +) -> Vec<(String, crate::compiler::models::TextLocation)> { + columns + .into_iter() + .map(|(name, [line, column, end_line, end_column])| { + ( + name.to_owned(), + crate::compiler::models::TextLocation { + line, + column, + end_line, + end_column, + }, + ) + }) + .collect() +} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs index 8f09c181d9..d197e9e0a3 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_declaration_references.rs @@ -1,4 +1,4 @@ -use crate::compiler::tests::helpers::scanned_references; +use crate::compiler::tests::helpers::{scanned_references, scanned_usages}; use crate::compiler::tests::test_types::DeclarationReferenceScanTestCase; #[test] @@ -87,3 +87,124 @@ fn given_sql_when_scanning_declaration_references_then_python_offsets_are_return ); } } + +#[test] +fn given_sql_when_walking_declaration_usages_then_malformed_references_are_skipped() { + let test_cases = [ + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @enum('status').OPEN, @const('rate')", + sql: "SELECT @enum('status').OPEN, @const('rate')", + expected_references: Some("enum:status.OPEN@7..27, const:rate@29..43"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @enum('status') , @const('rate')", + sql: "SELECT @enum('status') , @const('rate')", + expected_references: Some("const:rate@25..39"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @enum x 'unclosed", + sql: "SELECT @enum x 'unclosed", + expected_references: Some(""), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @enum x, @const('rate') 'unclosed", + sql: "SELECT @enum x, @const('rate') 'unclosed", + expected_references: Some("const:rate@16..30"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @const(rate), @enum(\"size\").S", + sql: "SELECT @const(rate), @enum(\"size\").S", + expected_references: Some("enum:size.S@21..36"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT '@enum(''status'').A', @const('rate') /* open", + sql: "SELECT '@enum(''status'').A', @const('rate') /* open", + expected_references: Some("const:rate@30..44"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @enumx('status').A, @const_x('rate'), @consté, @const", + sql: "SELECT @enumx('status').A, @const_x('rate'), @consté, @const('rate')", + expected_references: Some("const:rate@54..68"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT $$ @enum('status').A $$, @enum('size').M -- @const('r", + sql: "SELECT $$ @enum('status').A $$, @enum('size').M -- @const('rate')\n", + expected_references: Some("enum:size.M@32..47"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT x$$ @const('rate') $$", + sql: "SELECT x$$ @const('rate') $$", + expected_references: Some("const:rate@11..25"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @enum ( 'status' ) . OPEN, @enum('status').OPEN, @con", + sql: "SELECT @enum ( 'status' ) . OPEN, @enum('status').OPEN, @const('missing')", + expected_references: Some( + "enum:status.OPEN@7..32, enum:status.OPEN@34..54, const:missing@56..73", + ), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT `@const('rate')`, \"@enum('size').A\"", + sql: "SELECT `@const('rate')`, \"@enum('size').A\"", + expected_references: Some(""), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @const('rate'), @const('rate'), 'é' @enum('size').XL", + sql: "SELECT @const('rate'), @const('rate'), 'é' @enum('size').XL", + expected_references: Some("const:rate@7..21, const:rate@23..37, enum:size.XL@43..59"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @@const, @enum, @const", + sql: "SELECT @@const, @enum, @const", + expected_references: Some(""), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT 1", + sql: "SELECT 1", + expected_references: Some(""), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @enum('status').OPEN /* @const('rate') */ 'it''s' @co", + sql: "SELECT @enum('status').OPEN /* @const('rate') */ 'it''s' @const('rate')", + expected_references: Some("enum:status.OPEN@7..27, const:rate@57..71"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @const('rate' ", + sql: "SELECT @const('rate' ", + expected_references: Some(""), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT /* x @const('rate')", + sql: "SELECT /* x @const('rate')", + expected_references: Some(" | stop:comment"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT 'a @enum('status').A", + sql: "SELECT 'a @enum('status').A", + expected_references: Some(" | stop:quote"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @enum x 'oops @const('rate')", + sql: "SELECT @enum x 'oops @const('rate')", + expected_references: Some(" | stop:quote"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @const('rate') $tag$ @enum('size').S", + sql: "SELECT @const('rate') $tag$ @enum('size').S", + expected_references: Some("const:rate@7..21 | stop:quote"), + }, + DeclarationReferenceScanTestCase { + description: "usage walk: SELECT @enum(\"status\").OPEN, @const('rate') /* tail @enum('s", + sql: "SELECT @enum(\"status\").OPEN, @const('rate') /* tail @enum('size').S", + expected_references: Some("enum:status.OPEN@7..27, const:rate@29..43 | stop:comment"), + }, + ]; + for test_case in test_cases { + assert_eq!( + scanned_usages(test_case.sql).as_deref(), + test_case.expected_references, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_model_columns.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_model_columns.rs new file mode 100644 index 0000000000..0ff6ceda64 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_model_columns.rs @@ -0,0 +1,173 @@ +use crate::compiler::main::header_word_value::header_word_value; +use crate::compiler::main::model_header_column_locations::model_header_column_locations; +use crate::compiler::main::model_header_columns::model_header_columns; +use crate::compiler::main::model_output_columns::model_output_columns; +use crate::compiler::models::{HeaderColumnSpan, HeaderWord}; +use crate::compiler::tests::helpers::locations; +use crate::compiler::tests::test_types::{ + HeaderColumnsTestCase, HeaderWordTestCase, OutputColumnsTestCase, +}; + +const MODEL: &str = "MODEL (\n columns (\n order_id (type INTEGER),\n amount (description 'x')\n )\n);\n\ +SELECT o.order_id, sum(amount) total, x AS \"Net\", \u{17f}um AS \u{17f}elect, count(*)\nFROM orders o\n"; + +#[test] +fn given_bare_header_words_when_reading_then_values_match_python_projection() { + let test_cases = [ + HeaderWordTestCase { + description: "true", + word: "true", + expected_value: HeaderWord::Bool(true), + }, + HeaderWordTestCase { + description: "null", + word: "null", + expected_value: HeaderWord::Null, + }, + HeaderWordTestCase { + description: "-007", + word: "-007", + expected_value: HeaderWord::Integer("-7".to_owned()), + }, + HeaderWordTestCase { + description: "\u{663}\u{664}", + word: "\u{663}\u{664}", + expected_value: HeaderWord::Integer("34".to_owned()), + }, + HeaderWordTestCase { + description: "1.", + word: "1.", + expected_value: HeaderWord::Float(1.0), + }, + HeaderWordTestCase { + description: "+.5", + word: "+.5", + expected_value: HeaderWord::Float(0.5), + }, + HeaderWordTestCase { + description: "1.2.3", + word: "1.2.3", + expected_value: HeaderWord::Text, + }, + HeaderWordTestCase { + description: "-", + word: "-", + expected_value: HeaderWord::Text, + }, + ]; + + for test_case in test_cases { + assert_eq!( + header_word_value(test_case.word), + test_case.expected_value, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_model_files_when_locating_output_columns_then_simple_items_are_named() { + let test_cases = [ + OutputColumnsTestCase { + description: "qualified, implicit, quoted and long-s aliases", + contents: MODEL, + implicit_aliases: true, + expected_columns: vec![ + ("order_id", [7, 8, 7, 18]), + ("total", [7, 20, 7, 37]), + ("Net", [7, 39, 7, 49]), + ("\u{17f}elect", [7, 51, 7, 64]), + ], + }, + OutputColumnsTestCase { + description: "implicit aliases can be left out", + contents: MODEL, + implicit_aliases: false, + expected_columns: vec![ + ("order_id", [7, 8, 7, 18]), + ("Net", [7, 39, 7, 49]), + ("\u{17f}elect", [7, 51, 7, 64]), + ], + }, + OutputColumnsTestCase { + description: "a top-level UNION has no single output list", + contents: "MODEL (x 1);\nSELECT a FROM t UNION SELECT b FROM u\n", + implicit_aliases: true, + expected_columns: vec![], + }, + ]; + + for test_case in test_cases { + let columns = model_output_columns(test_case.contents, test_case.implicit_aliases); + + assert_eq!( + columns, + locations(test_case.expected_columns), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_model_header_when_locating_columns_then_spans_and_locations_match_python() { + let test_cases = [HeaderColumnsTestCase { + description: "two columns with metadata", + contents: MODEL, + header_start: 7, + header_end: 82, + expected_body: (19, 80), + expected_entries: vec![("order_id", [24, 47, 34, 46]), ("amount", [53, 77, 61, 76])], + expected_locations: vec![("order_id", [3, 5, 3, 13]), ("amount", [4, 5, 4, 11])], + }]; + + for test_case in test_cases { + let header: String = test_case + .contents + .chars() + .skip(test_case.header_start) + .take(test_case.header_end - test_case.header_start) + .collect(); + + let columns = model_header_columns(test_case.contents) + .expect("tokenizes") + .expect("declares columns"); + let named = + model_header_column_locations(test_case.contents, &header, test_case.header_start) + .expect("parses"); + + assert_eq!( + (columns.body_start, columns.body_end), + test_case.expected_body, + "{}", + test_case.description + ); + assert_eq!( + columns.entries, + test_case + .expected_entries + .into_iter() + .map( + |(name, [entry_start, entry_end, metadata_start, metadata_end])| ( + name.to_owned(), + HeaderColumnSpan { + entry_start, + entry_end, + metadata_start, + metadata_end + } + ) + ) + .collect::>(), + "{}", + test_case.description + ); + assert_eq!( + named, + locations(test_case.expected_locations), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_model_header_entry_span.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_model_header_entry_span.rs new file mode 100644 index 0000000000..08fe4bffa9 --- /dev/null +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_model_header_entry_span.rs @@ -0,0 +1,46 @@ +use crate::compiler::main::model_header_entry_span::model_header_entry_span; +use crate::compiler::tests::test_types::EntrySpanTestCase; + +#[test] +fn given_model_headers_when_locating_an_entry_then_code_point_spans_match_python() { + let test_cases = [ + EntrySpanTestCase { + description: "a first-line entry ends before its separator", + contents: "MODEL (cursor updated_at, description 'x');\nSELECT 1", + expected_span: Ok(Some((7, 24))), + }, + EntrySpanTestCase { + description: "offsets count code points after a non-ASCII value and trim whitespace", + contents: "MODEL (\n description \"caf\u{e9}\"\n cursor\tupdated_at\n);\nSELECT 1", + expected_span: Ok(Some((31, 48))), + }, + EntrySpanTestCase { + description: "a nested key of the same name is skipped", + contents: "MODEL (columns (cursor int) cursor y);\nSELECT 1", + expected_span: Ok(Some((28, 36))), + }, + EntrySpanTestCase { + description: "a key without a value has no span", + contents: "MODEL (cursor);\nSELECT 1", + expected_span: Ok(None), + }, + EntrySpanTestCase { + description: "text before MODEL is not a header", + contents: "\n\u{e9}MODEL (cursor x);", + expected_span: Ok(None), + }, + EntrySpanTestCase { + description: "an unterminated string is the tokenizer's error", + contents: "MODEL (cursor 'unterminated);\nSELECT 1", + expected_span: Err("unterminated single-quoted string at position 7"), + }, + ]; + for test_case in test_cases { + assert_eq!( + model_header_entry_span(test_case.contents, "cursor"), + test_case.expected_span.map_err(str::to_owned), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs index 043592355b..319db3016a 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs @@ -99,3 +99,32 @@ impl InterpolationHost for MapHost { .collect() } } + +pub(super) struct EntrySpanTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) expected_span: Result, &'static str>, +} + +pub(super) struct HeaderWordTestCase { + pub(super) description: &'static str, + pub(super) word: &'static str, + pub(super) expected_value: crate::compiler::models::HeaderWord, +} + +pub(super) struct OutputColumnsTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) implicit_aliases: bool, + pub(super) expected_columns: Vec<(&'static str, [usize; 4])>, +} + +pub(super) struct HeaderColumnsTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) header_start: usize, + pub(super) header_end: usize, + pub(super) expected_body: (usize, usize), + pub(super) expected_entries: Vec<(&'static str, [usize; 4])>, + pub(super) expected_locations: Vec<(&'static str, [usize; 4])>, +} diff --git a/fensu.toml b/fensu.toml index d2a0dd91ec..07273147c2 100644 --- a/fensu.toml +++ b/fensu.toml @@ -23,6 +23,16 @@ exclude = ["tests/e2e/fixtures/**"] [targets.python.thresholds] max_positional_args = 1 +[[targets.python.rule_exceptions]] +rule = "FFS120" +path = "src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py" +reason = "Preserve the existing positional project argument while adding an optional keyword-only projection flag for custom-rule hosts." + +[[targets.python.rule_exceptions]] +rule = "FFR001" +path = "src/sqlbuild/cli/commands/models.py" +reason = "The TYPE_CHECKING block contains annotation-only imports, avoiding executor and integration imports for read-only commands; runtime declarations remain structured models." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/adapters/bigquery/__init__.py" @@ -311,11 +321,13 @@ crate_names = [ "sqlbuild-analysis", "sqlbuild-attachments", "sqlbuild-cache", + "sqlbuild-cli", "sqlbuild-config", "sqlbuild-core", "sqlbuild-discovery", "sqlbuild-model-config", "sqlbuild-python", + "sqlbuild-refactor", "sqlbuild-render", "sqlbuild-rules", "sqlbuild-scopes", diff --git a/pyproject.toml b/pyproject.toml index a048a0af74..22f7ae477e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "pydantic-settings>=2,<3", "PyYAML>=6.0.2", "rich>=15.0.0", + "sqlbuild-cli==0.155.0", # x-release-please-version ] [project.optional-dependencies] @@ -55,8 +56,7 @@ rivers = [ ] [project.scripts] -sqb = "sqlbuild.cli.entry.main.entry:main" -sqlbuild = "sqlbuild.cli.entry.main.entry:main" +sqlbuild-python = "sqlbuild.cli.entry.main.launched:main" [dependency-groups] dev = [ @@ -143,3 +143,6 @@ cache-keys = [ { file = "crates/*/src/**/*.rs" }, { file = "src/sqlbuild/_native.pyi" }, ] + +[tool.uv.sources] +sqlbuild-cli = { path = "crates/sqlbuild-cli" } diff --git a/release-please-config.json b/release-please-config.json index daed61d50d..59d86efda8 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -74,6 +74,15 @@ "type": "toml", "path": "uv.lock", "jsonpath": "$.package[?(@.name.value == \"sqlbuild\")].version" + }, + { + "type": "toml", + "path": "uv.lock", + "jsonpath": "$.package[?(@.name.value == \"sqlbuild-cli\")].version" + }, + { + "type": "generic", + "path": "pyproject.toml" } ], "exclude-paths": [ diff --git a/scripts/compiler_differential/README.md b/scripts/compiler_differential/README.md index 4acdbf9eba..9ef654ead0 100644 --- a/scripts/compiler_differential/README.md +++ b/scripts/compiler_differential/README.md @@ -142,3 +142,11 @@ failure_case( ``` Then it runs `make compiler-baselines` to record the new case's golden. + +## Timing-dependent native metrics + +Raw captures retain `reuse_digested_files`, but the exact native-ownership gate excludes it. +The number of physical hashes depends on whether a file timestamp falls within the cache's +racy-write verification window. Different execution speeds can therefore change the count +without changing cache validity or output. Deterministic reuse and snapshot counters remain +gated, as do all actual Python fallback records. diff --git a/scripts/compiler_differential/_helpers/comparing/project_comparison.py b/scripts/compiler_differential/_helpers/comparing/project_comparison.py index fc49bc08f1..28d7434ee2 100644 --- a/scripts/compiler_differential/_helpers/comparing/project_comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/project_comparison.py @@ -132,7 +132,7 @@ def _corpus_differences(*, project: CorpusProject, run: EngineRun) -> list[Diffe problems: list[str] = [ f"`{outcome.label}` exit {outcome.exit_code}" for outcome in run.outcomes - if outcome.exit_code != 0 + if (outcome.exit_code != 0) != (outcome.label in project.expected.refused_commands) ] if errors: problems.append(f"compile errors {', '.join(errors)}") diff --git a/scripts/compiler_differential/_helpers/corpus/corpus.py b/scripts/compiler_differential/_helpers/corpus/corpus.py index 9fecf60b33..fbe187477f 100644 --- a/scripts/compiler_differential/_helpers/corpus/corpus.py +++ b/scripts/compiler_differential/_helpers/corpus/corpus.py @@ -7,6 +7,7 @@ from pathlib import Path from scripts.compiler_differential._helpers.corpus.failure_cases import all_failure_cases +from scripts.compiler_differential._helpers.refactor_corpus.refactor_cases import refactor_cases from scripts.compiler_differential.classes.dense_project import DenseProject from scripts.compiler_differential.classes.project_builder import ProjectBuilder from scripts.compiler_differential.constants import ( @@ -17,6 +18,7 @@ CORPUS_FAILURES, CORPUS_FIXTURES, CORPUS_SEEDS, + DIALECT_RICH_LINEAGE, DUCKDB_ADAPTER, EXAMPLE_ROOT, EXPECT_SUCCESS, @@ -95,6 +97,7 @@ def build_corpus( ) for case in all_failure_cases() ) + entries.extend(refactor_cases()) if CORPUS_DENSE in corpora: entries.extend( CorpusProject( @@ -156,14 +159,14 @@ def _seed_project(seed: int) -> CorpusProject: def _dialect_seed_project(*, seed: int, dialect: str) -> CorpusProject: - """Compile a seed's analysis blocks under another adapter's dialect; plan needs a warehouse.""" + """Compile a seed's analysis blocks, with rich lineage, under another adapter's dialect.""" generated: GeneratedProject = ProjectBuilder( seed, blocks=GENERATOR_DIALECT_BLOCKS, dialect=dialect ).build() return CorpusProject( name=f"seed/{seed}-{dialect}", - commands=(COLD_COMPILE,), + commands=(COLD_COMPILE, DIALECT_RICH_LINEAGE), expected=EXPECT_SUCCESS, writer=generated.write, seed_coverage=True, diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index b51f3e970a..4b6b977c44 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -24,7 +24,9 @@ EXPECT_SUCCESS, LEFT_SIDE, NATIVE_CODE_CONSTANT_PATTERN, + NATIVE_CODE_TEST_DIRECTORY, RENDER_CODE_SCAN_EXCLUDED, + RENDER_NATIVE_CODE_SCAN_ROOTS, RENDER_RAISED_CODES, WARNING_SEVERITY, ) @@ -141,6 +143,16 @@ def inline_compile_codes(compile_root: Path) -> dict[str, tuple[str, ...]]: code: str | None = _code_literal(node) if code is not None: found.setdefault(code, []).append(f"{relative}:{getattr(node, 'lineno', 0)}") + repo_root: Path = compile_root.parents[3] + for directory in RENDER_NATIVE_CODE_SCAN_ROOTS: + for path in sorted((repo_root / directory).rglob("*.rs")): + if NATIVE_CODE_TEST_DIRECTORY in path.parts: + continue + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + for code in NATIVE_CODE_CONSTANT_PATTERN.findall(line): + found.setdefault(code, []).append( + f"{path.relative_to(repo_root).as_posix()}:{line_number}" + ) return {code: tuple(locations) for code, locations in found.items()} diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index f39c7978f0..8611f2be48 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -406,6 +406,22 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), + failure_case( + name="rule-unused-cte-after-variable-and-macro", + expected_code="SQBRSQL005", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + '\n[vars]\nfloor = "2"\n' + + _RULES_CONFIG.format(codes="SQBRSQL005"), + "models/marts/_macros/cents.py": _ENGINE_ERROR_CENTS, + **mart_body_files( + 'WITH totals AS (SELECT customer_id, @cents("amount") AS cents, ' + '@@floor AS floor_cents FROM __ref("stg_orders")), spare AS (SELECT 1 AS one)\n' + "SELECT customer_id, SUM(cents) AS total_amount\n" + "FROM totals WHERE cents > floor_cents\nGROUP BY customer_id\n" + ), + }, + ), failure_case( name="rule-unused-cte", expected_code="SQBRSQL005", diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py index 8b048215c5..8ea156ca18 100644 --- a/scripts/compiler_differential/_helpers/coverage/render.py +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -22,6 +22,7 @@ ) from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( + COMPILE_INPUTS_CAPTURE_TYPE, RENDER_CALLABLE_CAPTURE_FIELDS, RENDER_DETAIL_KINDS, RENDER_HOOK_KEYS, @@ -31,6 +32,7 @@ ) from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_NAMES from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES _MACRO_CALL: re.Pattern[str] = re.compile(r"@([A-Za-z_]\w*)\(") _REFERENCE_CALL: re.Pattern[str] = re.compile(r"__(ref|source|seed)\(\s*\"([^\"]+)\"\s*\)") @@ -96,7 +98,12 @@ def render_capture_problems(capture_text: str) -> tuple[str, ...]: return capture_problems( capture_text=capture_text, - fields=tuple(field.name for field in dataclasses.fields(CompileProjectInputs)), + fields=tuple( + field.name + for field in dataclasses.fields(CompileProjectInputs) + if field.name + not in STAGE_CAPTURE_OMITTED_ATTRIBUTES.get(COMPILE_INPUTS_CAPTURE_TYPE, frozenset()) + ), callable_fields=RENDER_CALLABLE_CAPTURE_FIELDS, ) diff --git a/scripts/compiler_differential/_helpers/goldens/goldens.py b/scripts/compiler_differential/_helpers/goldens/goldens.py index e5e2d3f837..30d74f0011 100644 --- a/scripts/compiler_differential/_helpers/goldens/goldens.py +++ b/scripts/compiler_differential/_helpers/goldens/goldens.py @@ -17,15 +17,18 @@ ) from scripts.compiler_differential.constants import ( GOLDEN_CORPUS_PREFIXES, + GOLDEN_FULL_REPORT_LABEL_PREFIX, GOLDEN_LABEL, GOLDEN_MANIFEST_DROPPED_KEYS, GOLDEN_MANIFEST_RESOURCE_SECTIONS, GOLDEN_MISSING_HINT, GOLDEN_MODE_CHECK, GOLDEN_PATH_MASK, + GOLDEN_REPORT_KEY, GOLDEN_RESOURCE_DROPPED_FIELDS, GOLDEN_SEED_CORPUS, GOLDEN_SEED_RANGE_FILE, + GOLDEN_STDOUT_LABELS, GOLDEN_SUFFIX, GOLDEN_VERSION_MASK, GOLDEN_WORK_MASK, @@ -151,7 +154,7 @@ def _command( report: dict[str, object] | None = as_json_object(json.loads(stdout)) except json.JSONDecodeError: report = None - return { + command: dict[str, object] = { "label": outcome.label, "exit_code": "timed out" if outcome.timed_out else outcome.exit_code, "diagnostics": ( @@ -161,7 +164,15 @@ def _command( text=normalize_stderr(_masked(text=outcome.stderr, masked_paths=masked_paths)) ) ), + **( + {GOLDEN_REPORT_KEY: report} + if outcome.label.startswith(GOLDEN_FULL_REPORT_LABEL_PREFIX) + else {} + ), } + if outcome.label in GOLDEN_STDOUT_LABELS: + command["stdout"] = report if report is not None else _lines(text=stdout) + return command def _manifest(*, text: str | None, masked_paths: tuple[tuple[str, str], ...]) -> object | None: diff --git a/scripts/compiler_differential/_helpers/refactor_corpus/__init__.py b/scripts/compiler_differential/_helpers/refactor_corpus/__init__.py new file mode 100644 index 0000000000..570af3ed0a --- /dev/null +++ b/scripts/compiler_differential/_helpers/refactor_corpus/__init__.py @@ -0,0 +1 @@ +"""Refactoring cases that compare the edited tree and the JSON report.""" diff --git a/scripts/compiler_differential/_helpers/refactor_corpus/refactor_cases.py b/scripts/compiler_differential/_helpers/refactor_corpus/refactor_cases.py new file mode 100644 index 0000000000..5c23babbc3 --- /dev/null +++ b/scripts/compiler_differential/_helpers/refactor_corpus/refactor_cases.py @@ -0,0 +1,127 @@ +"""`sqb rename` and `sqb mv` cases that refactor the base project, then compile the result.""" + +from __future__ import annotations + +from scripts.compiler_differential.constants import ( + COLD_COMPILE, + FAILURE_BASE_FILES, + FAILURE_MART_PATH, + FAILURE_SOURCES_PATH, + FAILURE_STAGING_PATH, + GOLDEN_FULL_REPORT_LABEL_PREFIX, +) +from scripts.compiler_differential.models import ( + CorpusProject, + DifferentialCommand, + ExpectedOutcome, + WritableProject, +) + +_STAR_MART: str = """MODEL ( + description "Every staged order", +); + +SELECT * +FROM __ref("stg_orders") +""" +_TABLE_STAGING: str = """MODEL ( + description "Staged orders", + materialized table, +); + +SELECT order_id, customer_id, amount, status +FROM __source("raw_orders") +""" +_ANCHORED_SOURCES: str = """sources: + - name: raw_orders + description: &feed "Feeds __ref('stg_orders')." + expression: &orders >- + (SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, + 'placed' AS status) + columns: &order_columns + - name: order_id + type: !!str INTEGER + - name: customer_id + type: INTEGER + - name: amount + type: DOUBLE + - name: status + type: VARCHAR + - name: raw_orders_copy + description: *feed + expression: *orders + columns: *order_columns +""" +_HEADER_MART: str = """MODEL ( + description "Order totals per customer, from __ref('stg_orders')", + materialized table, +); + +SELECT customer_id, SUM(amount) AS total_amount +FROM __ref("stg_orders") +GROUP BY customer_id +""" + + +def _refactor(*arguments: str) -> DifferentialCommand: + return DifferentialCommand( + label=f"{GOLDEN_FULL_REPORT_LABEL_PREFIX}{arguments[0]}", + arguments=(*arguments, "--json"), + ) + + +def _case( + *, + name: str, + refactor: DifferentialCommand, + files: dict[str, str] | None = None, + refused: bool = False, +) -> CorpusProject: + return CorpusProject( + name=f"failure/refactor-{name}", + commands=(refactor, COLD_COMPILE), + expected=ExpectedOutcome(refused_commands=(refactor.label,) * refused), + writer=WritableProject(files={**FAILURE_BASE_FILES, **(files or {})}).write, + ) + + +def refactor_cases() -> tuple[CorpusProject, ...]: + """Return every refactoring case in a stable order.""" + + return ( + _case( + name="rename-model", + refactor=_refactor("rename", "stg_orders", "stg_order_lines"), + files={FAILURE_STAGING_PATH: _TABLE_STAGING, FAILURE_MART_PATH: _HEADER_MART}, + ), + _case( + name="rename-model-yaml-anchors", + refactor=_refactor("rename", "stg_orders", "stg_order_lines"), + files={FAILURE_SOURCES_PATH: _ANCHORED_SOURCES}, + ), + _case( + name="rename-model-dry-run", + refactor=_refactor("rename", "stg_orders", "stg_order_lines", "--dry-run"), + ), + _case(name="mv-model", refactor=_refactor("mv", "customer_totals", "models/finance/")), + _case( + name="rename-column", + refactor=_refactor("rename", "stg_orders.amount", "order_amount"), + ), + _case( + name="rename-column-cascade", + refactor=_refactor("rename", "stg_orders.amount", "order_amount", "--cascade"), + files={FAILURE_MART_PATH: _STAR_MART}, + ), + _case( + name="rename-column-star-refused", + refactor=_refactor("rename", "stg_orders.amount", "order_amount"), + files={FAILURE_MART_PATH: _STAR_MART}, + refused=True, + ), + _case( + name="rename-model-collision-refused", + refactor=_refactor("rename", "stg_orders", "customer_totals"), + refused=True, + ), + ) diff --git a/scripts/compiler_differential/_helpers/running/native_fallbacks.py b/scripts/compiler_differential/_helpers/running/native_fallbacks.py index 625c5ed66f..f62d11f070 100644 --- a/scripts/compiler_differential/_helpers/running/native_fallbacks.py +++ b/scripts/compiler_differential/_helpers/running/native_fallbacks.py @@ -17,6 +17,7 @@ NATIVE_FALLBACK_MODE_UPDATE, NATIVE_FALLBACK_PROJECT_CORPUS, NATIVE_FALLBACK_SITE_SEPARATOR, + NATIVE_TIMING_DEPENDENT_METRICS, ) from scripts.compiler_differential.models import ( AllowList, @@ -53,13 +54,15 @@ def native_fallback_gate( def observed_fallbacks( *, comparisons: list[ProjectComparison], engines: tuple[str, ...] ) -> dict[FallbackKey, dict[str, int]]: - """Sum every engine's fallback and analysis deferral records by entry and corpus.""" + """Sum fallbacks and deterministic native answers, excluding timing-dependent metrics.""" counts: Counter[tuple[FallbackKey, str]] = Counter() for comparison in comparisons: corpus: str = comparison.project.split("/", 1)[0] for engine, records in zip(engines, comparison.records or (), strict=False): for (site, kind), count in records.fallbacks.items(): + if (site, kind) in NATIVE_TIMING_DEPENDENT_METRICS: + continue stage: str = site.split(NATIVE_FALLBACK_SITE_SEPARATOR, 1)[0] counts[((engine, stage, site, kind), corpus)] += count for (kind, site), count in records.deferrals.items(): diff --git a/scripts/compiler_differential/classes/analysis_features.py b/scripts/compiler_differential/classes/analysis_features.py index 9240dc67dc..93c6a1721b 100644 --- a/scripts/compiler_differential/classes/analysis_features.py +++ b/scripts/compiler_differential/classes/analysis_features.py @@ -8,6 +8,7 @@ from scripts.compiler_differential.constants import ( GENERATOR_ANALYSIS_FOLDER, GENERATOR_ANALYSIS_MODE_COMMANDS, + GENERATOR_ANALYSIS_MODEL, GENERATOR_ANALYSIS_TEST_FOLDER, GENERATOR_OPEN_SOURCE, GENERATOR_SELECTED_MODEL, @@ -354,17 +355,27 @@ def _analysis_modes(self) -> None: ), ) self.commands = tuple( - _selected_command(label=label, arguments=arguments, model=base.name) + _selected_command( + label=label, + arguments=arguments, + model=base.name, + analysis_model=f"cte_facts_{base.name}", + ) for label, arguments in GENERATOR_ANALYSIS_MODE_COMMANDS ) -def _selected_command(*, label: str, arguments: tuple[str, ...], model: str) -> DifferentialCommand: - """Return one analysis-mode command with the selected model substituted.""" +def _selected_command( + *, label: str, arguments: tuple[str, ...], model: str, analysis_model: str +) -> DifferentialCommand: + """Return one analysis-mode command with the selected and analysis models substituted.""" return DifferentialCommand( label=label, arguments=tuple( - model if argument == GENERATOR_SELECTED_MODEL else argument for argument in arguments + argument.replace(GENERATOR_ANALYSIS_MODEL, analysis_model).replace( + GENERATOR_SELECTED_MODEL, model + ) + for argument in arguments ), ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 80a567e030..56d36f9427 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -24,6 +24,9 @@ NATIVE_FALLBACK_MAX_COUNTS_FIELD: str = "max_counts" NATIVE_FALLBACK_COUNTS_FIELD: str = "counts" NATIVE_FALLBACK_PROJECT_CORPUS: str = "project" +NATIVE_TIMING_DEPENDENT_METRICS: frozenset[tuple[str, str]] = frozenset( + {("compile_outputs.native", "reuse_digested_files")} +) NATIVE_FALLBACK_DEFERRAL_STAGES: dict[str, str] = { "analysis_session": "model_analysis", "contracts/columns.py": "contracts", @@ -48,6 +51,8 @@ GOLDEN_SEED_RANGE_FILE: str = "seed_range.toml" GOLDEN_SUFFIX: str = ".json" GOLDEN_LABEL: str = "golden" +GOLDEN_FULL_REPORT_LABEL_PREFIX: str = "refactor-" +GOLDEN_REPORT_KEY: str = "report" GOLDEN_MANIFEST_DROPPED_KEYS: frozenset[str] = frozenset({"metadata", "child_map"}) GOLDEN_MANIFEST_RESOURCE_SECTIONS: tuple[str, ...] = ("nodes", "sources", "macros") GOLDEN_RESOURCE_DROPPED_FIELDS: frozenset[str] = frozenset( @@ -318,10 +323,31 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: block for block in GENERATOR_ANALYSIS_BLOCKS if block not in GENERATOR_DIALECT_EXCLUDED_BLOCKS ) GENERATOR_SELECTED_MODEL: str = "{model}" +GENERATOR_ANALYSIS_MODEL: str = "{analysis_model}" +GENERATOR_RICH_LINEAGE_ARGUMENTS: tuple[str, ...] = ("compile", "--json", "--lineage-mode", "rich") GENERATOR_ANALYSIS_MODE_COMMANDS: tuple[tuple[str, tuple[str, ...]], ...] = ( ("compile-select", ("compile", "--json", "--select", GENERATOR_SELECTED_MODEL)), ("compile-no-analysis", ("compile", "--json", "--no-sql-analysis")), - ("compile-rich-lineage", ("compile", "--json", "--lineage-mode", "rich")), + ("compile-rich-lineage", GENERATOR_RICH_LINEAGE_ARGUMENTS), + ( + "lineage-rich-upstream", + ("lineage", "--json", f"{GENERATOR_ANALYSIS_MODEL}.amount", "--direction", "upstream"), + ), + ( + "lineage-rich-downstream", + ("lineage", "--json", f"{GENERATOR_SELECTED_MODEL}.amount", "--direction", "downstream"), + ), + ( + "lineage-fast-upstream", + ("lineage", "--json", "--mode", "fast", f"{GENERATOR_ANALYSIS_MODEL}.amount"), + ), + ("lineage-relations", ("lineage", "--json", GENERATOR_SELECTED_MODEL, "--direction", "both")), +) +GOLDEN_STDOUT_LABELS: frozenset[str] = frozenset( + label for label, _ in GENERATOR_ANALYSIS_MODE_COMMANDS if label.startswith("lineage-") +) +DIALECT_RICH_LINEAGE: DifferentialCommand = DifferentialCommand( + label="compile-rich-lineage", arguments=GENERATOR_RICH_LINEAGE_ARGUMENTS ) GENERATOR_CHANNEL_ENV_VAR: str = "SQB_DIFFERENTIAL_CHANNEL" GENERATOR_MISSING_ENV_VAR: str = "SQB_DIFFERENTIAL_UNSET" @@ -469,6 +495,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: RENDER_STAGE_CAPTURE_SUFFIX: str = "-compile_project_inputs.json" RENDER_NON_COLLECTION_FIELDS: frozenset[str] = frozenset( { + "native_project", "project_config", "local_config", "discovered_inputs", @@ -560,7 +587,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: } RENDER_CONFIG_ONLY_KINDS: dict[str, str] = {} DIAGNOSTIC_CODE_PATTERN: re.Pattern[str] = re.compile(r"[A-Z][0-9]{3}") -RENDER_RAISED_CODES: frozenset[str] = frozenset({"C214", "C216", "K012"}) +RENDER_RAISED_CODES: frozenset[str] = frozenset({"C214", "C216", "K012", "P004", "P010", "P011"}) RENDER_CODE_SCAN_EXCLUDED: dict[str, str] = { "_helpers/assembly": "semantic and metadata validation of the assembled CompiledProject (M4)", "_helpers/analysis": "SQL analysis of rendered SQL, after the CompileProjectInputs frontier", @@ -600,8 +627,10 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: } ANALYSIS_STAGE_CAPTURE_SUFFIX: str = "-compiled_project.json" COMPILED_PROJECT_CAPTURE_TYPE: str = "sqlbuild.compiler.compile.models:CompiledProject" +COMPILE_INPUTS_CAPTURE_TYPE: str = "sqlbuild.compiler.compile.models:CompileProjectInputs" ANALYSIS_NON_COLLECTION_FIELDS: frozenset[str] = frozenset( { + "native_project", "run_id", "effective_target_name", "effective_connection", @@ -686,7 +715,16 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "sql_analysis", ) ANALYSIS_NATIVE_CODE_SCAN_ROOTS: tuple[str, ...] = ( + "crates/sqlbuild-analysis/src/semantic_checks/constants.rs", + "crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs", "crates/sqlbuild-analysis/src/contracts/constants.rs", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs", +) +NATIVE_CODE_TEST_DIRECTORY: str = "tests" +RENDER_NATIVE_CODE_SCAN_ROOTS: tuple[str, ...] = ( + "crates/sqlbuild-attachments/src", + "crates/sqlbuild-render/src", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project", ) NATIVE_CODE_CONSTANT_PATTERN: re.Pattern[str] = re.compile( r'const [A-Z0-9_]*_CODE: &str = "([A-Z][0-9]{3})";' diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index d47a9f3212..ba357f1ea0 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -25,6 +25,8 @@ class ExpectedOutcome: error_code: str | None = None succeeding_commands: tuple[str, ...] = () warning_code: str | None = None + refused_commands: tuple[str, ...] = () + """Labels of commands that must exit non-zero in an otherwise successful project.""" @dataclass(frozen=True) diff --git a/scripts/compiler_differential/native_fallbacks.toml b/scripts/compiler_differential/native_fallbacks.toml index 408b883a8a..ceed6c0217 100644 --- a/scripts/compiler_differential/native_fallbacks.toml +++ b/scripts/compiler_differential/native_fallbacks.toml @@ -6,33 +6,299 @@ seed_start = 0 seeds = 12 +[[entry]] +engine = "native" +stage = "attachments" +site = "attachments.native" +kind = "cursor_intrinsic_rejections" +counts = { failure = 3, seed = 257 } + +[[entry]] +engine = "native" +stage = "compile_lint_inputs" +site = "compile_lint_inputs.native" +kind = "lint_expansions" +counts = { failure = 4 } + +[[entry]] +engine = "native" +stage = "compile_outputs" +site = "compile_outputs.native" +kind = "artifact_files" +counts = { example = 151, failure = 501, fixture = 141, seed = 1188 } + +[[entry]] +engine = "native" +stage = "compile_outputs" +site = "compile_outputs.native" +kind = "json_reports" +counts = { example = 9, failure = 349, fixture = 9, seed = 40 } + +[[entry]] +engine = "native" +stage = "compile_outputs" +site = "compile_outputs.native" +kind = "reuse_records" +counts = { example = 3, fixture = 2, seed = 11 } + +[[entry]] +engine = "native" +stage = "compile_outputs" +site = "compile_outputs.native" +kind = "reuse_snapshot_paths" +counts = { example = 73, fixture = 73, seed = 1452 } + [[entry]] engine = "native" stage = "macro_calls" site = "macro_calls.mocked_evaluation" kind = "test_mock" -counts = { seed = 29 } +counts = { seed = 43 } + +[[entry]] +engine = "native" +stage = "model_config" +site = "model_config.native" +kind = "config_builds" +counts = { example = 56, failure = 225, fixture = 52, seed = 1086 } + +[[entry]] +engine = "native" +stage = "model_config" +site = "model_config.native" +kind = "header_metadata_parses" +counts = { example = 56, failure = 357, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native" +stage = "model_config" +site = "model_config.native" +kind = "validations" +counts = { example = 56, failure = 361, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native" +stage = "model_loop" +site = "model_loop.native" +kind = "cursor_intrinsic_validations" +counts = { example = 56, failure = 361, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native" +stage = "model_loop" +site = "model_loop.native" +kind = "declaration_contexts" +counts = { example = 12, failure = 56, fixture = 12, seed = 681 } + +[[entry]] +engine = "native" +stage = "model_loop" +site = "model_loop.native" +kind = "declaration_expansions" +counts = { example = 56, failure = 361, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native" +stage = "model_loop" +site = "model_loop.native" +kind = "native_model_inputs" +counts = { example = 56, failure = 361, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native" +stage = "refactoring" +site = "refactoring.native" +kind = "declaration_moves" +counts = { failure = 1 } + +[[entry]] +engine = "native" +stage = "refactoring" +site = "refactoring.native" +kind = "migration_edits" +counts = { failure = 1 } + +[[entry]] +engine = "native" +stage = "refactoring" +site = "refactoring.native" +kind = "planned_edits" +counts = { failure = 11 } + +[[entry]] +engine = "native" +stage = "rules_request" +site = "rules_request.native" +kind = "models" +counts = { example = 56, failure = 100, fixture = 52, seed = 852 } + +[[entry]] +engine = "native" +stage = "rules_request" +site = "rules_request.native" +kind = "sql_scenarios" +counts = { example = 8, fixture = 12, seed = 39 } + +[[entry]] +engine = "native" +stage = "rules_request" +site = "rules_request.native" +kind = "sql_tests" +counts = { example = 16, failure = 3, fixture = 20, seed = 160 } [[entry]] engine = "native" stage = "sql_test_glue" site = "sql_test_glue.macro_mock_queries" kind = "extension_callback" -counts = { seed = 29 } +counts = { seed = 43 } + +[[entry]] +engine = "native-preview" +stage = "attachments" +site = "attachments.native" +kind = "cursor_intrinsic_rejections" +counts = { failure = 3, seed = 257 } + +[[entry]] +engine = "native-preview" +stage = "compile_lint_inputs" +site = "compile_lint_inputs.native" +kind = "lint_expansions" +counts = { failure = 4 } + +[[entry]] +engine = "native-preview" +stage = "compile_outputs" +site = "compile_outputs.native" +kind = "artifact_files" +counts = { example = 151, failure = 501, fixture = 141, seed = 1188 } + +[[entry]] +engine = "native-preview" +stage = "compile_outputs" +site = "compile_outputs.native" +kind = "json_reports" +counts = { example = 9, failure = 349, fixture = 9, seed = 40 } + +[[entry]] +engine = "native-preview" +stage = "compile_outputs" +site = "compile_outputs.native" +kind = "reuse_records" +counts = { example = 3, fixture = 2, seed = 11 } + +[[entry]] +engine = "native-preview" +stage = "compile_outputs" +site = "compile_outputs.native" +kind = "reuse_snapshot_paths" +counts = { example = 73, fixture = 73, seed = 1452 } [[entry]] engine = "native-preview" stage = "macro_calls" site = "macro_calls.mocked_evaluation" kind = "test_mock" -counts = { seed = 29 } +counts = { seed = 43 } + +[[entry]] +engine = "native-preview" +stage = "model_config" +site = "model_config.native" +kind = "config_builds" +counts = { example = 56, failure = 225, fixture = 52, seed = 1086 } + +[[entry]] +engine = "native-preview" +stage = "model_config" +site = "model_config.native" +kind = "header_metadata_parses" +counts = { example = 56, failure = 357, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native-preview" +stage = "model_config" +site = "model_config.native" +kind = "validations" +counts = { example = 56, failure = 361, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native-preview" +stage = "model_loop" +site = "model_loop.native" +kind = "cursor_intrinsic_validations" +counts = { example = 56, failure = 361, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native-preview" +stage = "model_loop" +site = "model_loop.native" +kind = "declaration_contexts" +counts = { example = 12, failure = 56, fixture = 12, seed = 681 } + +[[entry]] +engine = "native-preview" +stage = "model_loop" +site = "model_loop.native" +kind = "declaration_expansions" +counts = { example = 56, failure = 361, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native-preview" +stage = "model_loop" +site = "model_loop.native" +kind = "native_model_inputs" +counts = { example = 56, failure = 361, fixture = 52, seed = 1294 } + +[[entry]] +engine = "native-preview" +stage = "refactoring" +site = "refactoring.native" +kind = "declaration_moves" +counts = { failure = 1 } + +[[entry]] +engine = "native-preview" +stage = "refactoring" +site = "refactoring.native" +kind = "migration_edits" +counts = { failure = 1 } + +[[entry]] +engine = "native-preview" +stage = "refactoring" +site = "refactoring.native" +kind = "planned_edits" +counts = { failure = 11 } + +[[entry]] +engine = "native-preview" +stage = "rules_request" +site = "rules_request.native" +kind = "models" +counts = { example = 56, failure = 100, fixture = 52, seed = 852 } + +[[entry]] +engine = "native-preview" +stage = "rules_request" +site = "rules_request.native" +kind = "sql_scenarios" +counts = { example = 8, fixture = 12, seed = 39 } + +[[entry]] +engine = "native-preview" +stage = "rules_request" +site = "rules_request.native" +kind = "sql_tests" +counts = { example = 16, failure = 3, fixture = 20, seed = 160 } [[entry]] engine = "native-preview" stage = "sql_test_glue" site = "sql_test_glue.macro_mock_queries" kind = "extension_callback" -counts = { seed = 29 } +counts = { seed = 43 } [[entry]] engine = "python" diff --git a/scripts/native_layering/_helpers/versions.py b/scripts/native_layering/_helpers/versions.py index c9c05d26c2..1b3f6940d4 100644 --- a/scripts/native_layering/_helpers/versions.py +++ b/scripts/native_layering/_helpers/versions.py @@ -9,6 +9,7 @@ from scripts.native_layering.constants import ( CARGO_MANIFEST, + CLI_DISTRIBUTION_PIN_PREFIX, PYPROJECT_MANIFEST, RELEASE_MANIFEST, RELEASE_MANIFEST_PACKAGE, @@ -28,6 +29,14 @@ def get_native_version_errors(root: Path) -> tuple[str, ...]: .get("version"), f"{PYPROJECT_MANIFEST} [project] version": pyproject.get("project", {}).get("version"), f"{RELEASE_MANIFEST} version": release.get(RELEASE_MANIFEST_PACKAGE), + f"{PYPROJECT_MANIFEST} sqlbuild-cli dependency": next( + ( + dependency.removeprefix(CLI_DISTRIBUTION_PIN_PREFIX) + for dependency in pyproject.get("project", {}).get("dependencies", ()) + if dependency.startswith(CLI_DISTRIBUTION_PIN_PREFIX) + ), + None, + ), } if len(set(versions.values())) == 1: return () diff --git a/scripts/native_layering/constants.py b/scripts/native_layering/constants.py index 6cf6a3b6f8..34b20f736e 100644 --- a/scripts/native_layering/constants.py +++ b/scripts/native_layering/constants.py @@ -8,3 +8,4 @@ PYPROJECT_MANIFEST: str = "pyproject.toml" RELEASE_MANIFEST: str = ".release-please-manifest.json" RELEASE_MANIFEST_PACKAGE: str = "." +CLI_DISTRIBUTION_PIN_PREFIX: str = "sqlbuild-cli==" diff --git a/scripts/python_char_table/constants.py b/scripts/python_char_table/constants.py index 95bc78d53a..ce09018898 100644 --- a/scripts/python_char_table/constants.py +++ b/scripts/python_char_table/constants.py @@ -6,7 +6,7 @@ TABLE_FILE_PREFIX: str = "unicode_" TABLE_FILE_SUFFIX: str = ".in" MAX_CODE_POINT: int = 0x10FFFF -TABLE_METHODS: tuple[str, ...] = ("isalnum", "isalpha", "isdecimal") +TABLE_METHODS: tuple[str, ...] = ("isalnum", "isalpha", "isdecimal", "isprintable") MAPPING_METHODS: tuple[str, ...] = ("casefold", "upper") IGNORECASE_KEY_TABLE: str = "re_ignorecase_keys" SURROGATE_RANGE: range = range(0xD800, 0xE000) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index ab3b35a1f5..45ce79f9d1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -120,7 +120,7 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as `settings_customise_sources` or command-line parsing. The reason is logged at debug level. SQLBuild keeps one stored result per project and target under -`target/cache/compiler-native-v1/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is +`target/cache/compiler-native-v1/project-reuse-v2/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the report is ready. When it has to hash a lot of data or the project has very many files, compile prints `Recording compile for reuse...` and a completion line to stderr before the report. @@ -129,7 +129,8 @@ prints `Recording compile for reuse...` and a completion line to stderr before t Any change runs the whole compile again: every model, test, and audit is rendered again, and contracts and semantic validation run for the whole project. No render is carried over from the -previous compile; rendering runs natively and does not run unchanged macro calls again. A few +previous compile; rendering runs natively and does not run unchanged macro calls again. Every +model's SQL references are scanned again too; no reference scan is kept between compiles. A few caches under `target/cache/` shorten the rest: - An unchanged model takes its column analysis from `model-analyses.bin`. Entries are keyed by diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 51920c796c..3e8573a391 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1,14 +1,83 @@ """Private SQLBuild native engine bindings.""" from collections.abc import Callable, Mapping, Sequence +from pathlib import Path from typing import Any, TypedDict +from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds +from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity +from sqlbuild.compiler.compile.models import ( + CompileAuditInput, + CompiledAudit, + CompiledFunction, + CompiledLineageColumnFact, + CompiledObjectKey, + CompiledRelationLocation, + CompiledSeed, + CompiledSource, + CompiledSqlExpansion, + CompiledSqlScenario, + CompiledSqlTest, + CompileModelConfig, + CompilerDiagnostic, + CompileSeedInput, + CompileSourceInput, + CompileSqlFunctionInput, + CompileSqlReference, + CompileSqlScenarioInput, + CompileSqlTestInput, + DeclarationScopeBuild, + DynamicColumnContractProof, + FunctionArgument, + FunctionReturnColumn, + InferredColumn, + LoadedMacro, + MacroContext, +) +from sqlbuild.compiler.compile.types import ( + AttachedAuditTargetKind, + CompiledModelView, + FunctionLanguage, +) +from sqlbuild.compiler.discovery.models import ( + ConstantDeclaration, + DiscoveredAuditBlock, + DiscoveredAuditFile, + DiscoveredHookFunction, + DiscoveredLoaderFunction, + DiscoveredMaterializationFile, + DiscoveredProjectInputs, + DiscoveredPythonFunctionFile, + DiscoveredSchemaFile, + DiscoveredSeedFile, + DiscoveredSourceFile, + DiscoveredSqlFunctionFile, + DiscoveredSqlHookFile, + DiscoveredSqlModelFile, + EnumDeclaration, +) +from sqlbuild.compiler.references.models import LiteralSqlRelation +from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver +from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestAssemblyRequest, NativeSqlTestChainRequest, NativeSqlTestPlanningRequest, ) from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestAssemblyRow, NativeSqlTestPlanRow +from sqlbuild.lint.types import RelationKeyRow +from sqlbuild.spec.contracts.models import ( + LocalConfig, + ProjectConfig, + ScenarioConfig, + SchemaModelEntry, + SchemaSeedEntry, + SettingsConfig, + SourceEntry, + SourceLocation, + TargetConfig, +) BUILD_IDENTITY: str @@ -21,17 +90,716 @@ class LintPreparationRequest(TypedDict): prior_sites: list[int] dialect: str -class ParsedRulesRequest: - """Decoded built-in rules request held natively until it is evaluated once.""" - def analyze_column_references_json(request_json: str) -> str: ... def catalogue_json() -> str: ... -def evaluate_json(request_json: str) -> str: ... -def parse_rules_parts( - request_json: bytes, model_jsons: list[bytes], model_digests: list[str] -) -> ParsedRulesRequest: ... -def evaluate_parsed_rules(parsed: ParsedRulesRequest) -> str: ... -def finalize_rule_findings_json(request_json: str) -> str: ... + +class NativeRulesRequest: + """A natively built rules request held until it is evaluated once.""" + +def build_rules_request( + project: dict[str, object], + models: tuple[NativeCompiledProject, list[str]], + sql_tests: list[tuple[object, ...]], + scenarios: list[tuple[object, ...]], +) -> tuple[NativeRulesRequest, tuple[int, int, int]]: ... +def evaluate_rules_request( + request: NativeRulesRequest, +) -> tuple[ + list[tuple[bool, str, str, int, int, str, str]], list[str], int, int, int, int, bool +]: ... +def emit_orjson_report(report: object) -> str: ... +def emit_json_report(report: object) -> str: ... + +type _GraphKey = tuple[str, str] +type _GraphEdges = list[tuple[_GraphKey, list[_GraphKey]]] +type _GraphIndexes = tuple[ + list[tuple[str, _GraphKey]], + _GraphEdges, + _GraphEdges, + list[tuple[str, list[_GraphKey]]], + list[tuple[_GraphKey, str]], +] +type _SelectorFailure = tuple[str, str, str | None] +type _SelectorKeys = tuple[list[_GraphKey] | None, _SelectorFailure | None] + +class NativeModelInput: + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... + @property + def attached_schema_name(self) -> str | None: ... + @property + def relation_key_row(self) -> RelationKeyRow: ... + @property + def model_file(self) -> DiscoveredSqlModelFile: ... + @property + def config(self) -> CompileModelConfig: ... + @property + def query_sql(self) -> str: ... + @property + def macro_source_sql(self) -> str: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def schema_entry(self) -> SchemaModelEntry | None: ... + @property + def schema_file(self) -> DiscoveredSchemaFile | None: ... + @property + def sql_validation_enabled(self) -> bool: ... + @property + def rejected_sql_analysis_opt_out(self) -> SourceLocation | None: ... + @property + def enum_declarations(self) -> tuple[EnumDeclaration, ...]: ... + @property + def constant_declarations(self) -> tuple[ConstantDeclaration, ...]: ... + @property + def enum_columns(self) -> dict[str, EnumDeclaration]: ... + @property + def macro_deps(self) -> tuple[str, ...]: ... + @property + def macro_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def sql_expansion(self) -> CompiledSqlExpansion | None: ... + @property + def lint_expansion(self) -> CompiledSqlExpansion | None: ... + def compiled_model( + self, + arguments: tuple[ + list[tuple[str, str]], + tuple[NativeModelAnalysisSession, int] | None, + list[tuple[str, str | None, str]] | None, + tuple[ + bool, + list[tuple[str, str | None, str]], + list[tuple[str, str | None]], + list[str], + str | None, + bool, + ] + | None, + list[str], + ], + ) -> NativeCompiledModel: ... + +class NativeCompiledModel: + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... + @property + def relation_key_row(self) -> RelationKeyRow: ... + @property + def key_row(self) -> tuple[str, str]: ... + @property + def dep_rows(self) -> list[tuple[str, str]]: ... + @property + def destination_row(self) -> tuple[str | None, str | None, str]: ... + @property + def inferred_rows(self) -> list[tuple[str, str | None, str]] | None: ... + @property + def schema_column_rows(self) -> list[tuple[str, str | None]] | None: ... + @property + def has_dynamic_columns(self) -> bool: ... + @property + def output_column_count(self) -> int: ... + @property + def key(self) -> CompiledObjectKey: ... + @property + def deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def relative_path(self) -> Path: ... + @property + def query_sql(self) -> str: ... + @property + def config(self) -> CompileModelConfig: ... + @property + def destination(self) -> CompiledRelationLocation: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def schema_entry(self) -> SchemaModelEntry | None: ... + @property + def inferred_columns(self) -> tuple[InferredColumn, ...] | None: ... + @property + def fast_lineage_columns(self) -> Sequence[CompiledLineageColumnFact] | None: ... + @property + def fast_lineage_has_star(self) -> bool: ... + @property + def fast_lineage_star_resolved(self) -> bool: ... + @property + def authored_sql(self) -> str: ... + @property + def authored_query_sql(self) -> str: ... + @property + def output_column_locations(self) -> dict[str, SourceLocation]: ... + @property + def extract_implicit_alias_columns(self) -> bool: ... + @property + def macro_deps(self) -> tuple[str, ...]: ... + @property + def enum_declarations(self) -> tuple[EnumDeclaration, ...]: ... + @property + def constant_declarations(self) -> tuple[ConstantDeclaration, ...]: ... + @property + def enum_columns(self) -> dict[str, EnumDeclaration]: ... + @property + def binding_diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... + @property + def binding_validated(self) -> bool: ... + @property + def dynamic_column_contract(self) -> DynamicColumnContractProof | None: ... + @property + def unchecked_output_columns(self) -> frozenset[str]: ... + @property + def rejected_sql_analysis_opt_out(self) -> SourceLocation | None: ... + def with_binding_diagnostics( + self, diagnostics: tuple[CompilerDiagnostic, ...] + ) -> NativeCompiledModel: ... + def with_type_recovery( + self, diagnostics: tuple[CompilerDiagnostic, ...], poisoned: list[str] + ) -> NativeCompiledModel: ... + def with_target_defaults( + self, defaults: tuple[str | None, str | None], render: Callable[..., str | None] + ) -> NativeCompiledModel: ... + +class NativeInputsView: + @property + def project_config(self) -> ProjectConfig: ... + @property + def local_config(self) -> LocalConfig: ... + @property + def discovered_inputs(self) -> DiscoveredProjectInputs: ... + @property + def sql_lexical_syntax(self) -> SqlLexicalSyntax: ... + @property + def run_id(self) -> str: ... + @property + def effective_target_name(self) -> str | None: ... + @property + def effective_target(self) -> TargetConfig | None: ... + @property + def compile_cache_dir(self) -> Path | None: ... + @property + def effective_connection(self) -> dict[str, object]: ... + @property + def effective_settings(self) -> SettingsConfig: ... + @property + def no_sql_validation(self) -> bool: ... + @property + def effective_vars(self) -> dict[str, object]: ... + @property + def macro_context(self) -> MacroContext | None: ... + @property + def loaded_macros(self) -> dict[str, LoadedMacro]: ... + @property + def public_enums(self) -> dict[str, EnumDeclaration]: ... + @property + def public_constants(self) -> dict[str, ConstantDeclaration]: ... + @property + def model_inputs(self) -> tuple[NativeModelInput, ...]: ... + @property + def seed_inputs(self) -> tuple[CompileSeedInput, ...]: ... + @property + def source_inputs(self) -> tuple[CompileSourceInput, ...]: ... + @property + def sql_function_inputs(self) -> tuple[CompileSqlFunctionInput, ...]: ... + @property + def test_inputs(self) -> tuple[CompileSqlTestInput, ...]: ... + @property + def scenario_inputs(self) -> tuple[CompileSqlScenarioInput, ...]: ... + @property + def audit_inputs(self) -> tuple[CompileAuditInput, ...]: ... + @property + def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... + @property + def external_sql_reference_resolver(self) -> ExternalSqlReferenceResolver | None: ... + @property + def scope_index(self) -> ScopeIndex: ... + @property + def declaration_scope(self) -> DeclarationScopeBuild | None: ... + @property + def native_project(self) -> NativeCompiledProject | None: ... + def with_changes(self, **changes: object) -> NativeInputsView: ... + def fields(self) -> dict[str, Any]: ... + +class NativeSeedInput: + @property + def seed_file(self) -> DiscoveredSeedFile: ... + @property + def schema_entry(self) -> SchemaSeedEntry: ... + @property + def schema_file(self) -> DiscoveredSchemaFile: ... + +class NativeCompiledSeed: + @property + def key(self) -> CompiledObjectKey: ... + @property + def deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def seed_file(self) -> DiscoveredSeedFile: ... + @property + def schema_entry(self) -> SchemaSeedEntry: ... + @property + def schema_file(self) -> DiscoveredSchemaFile: ... + @property + def destination(self) -> CompiledRelationLocation: ... + @property + def external(self) -> bool: ... + def with_target_defaults( + self, + defaults: tuple[str | None, str | None], + render: Callable[..., str | None], + ) -> NativeCompiledSeed: ... + +class NativeSourceInput: + @property + def source_entry(self) -> SourceEntry: ... + @property + def source_file(self) -> DiscoveredSourceFile: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def assembly_row(self) -> tuple[str, bool, str | None, str | None]: ... + +class NativeCompiledSource: + @property + def key(self) -> CompiledObjectKey: ... + @property + def deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def source_entry(self) -> SourceEntry: ... + @property + def source_file(self) -> DiscoveredSourceFile: ... + +class NativeFunctionInput: + @property + def function_file(self) -> DiscoveredSqlFunctionFile | DiscoveredPythonFunctionFile: ... + @property + def name(self) -> str: ... + @property + def arguments(self) -> tuple[FunctionArgument, ...]: ... + @property + def returns(self) -> str: ... + @property + def body_sql(self) -> str: ... + @property + def return_columns(self) -> tuple[FunctionReturnColumn, ...]: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def language(self) -> FunctionLanguage: ... + @property + def runtime_version(self) -> str | None: ... + @property + def entry_point(self) -> str | None: ... + @property + def packages(self) -> tuple[str, ...]: ... + @property + def tags(self) -> tuple[str, ...]: ... + @property + def description(self) -> str | None: ... + @property + def database(self) -> str | None: ... + @property + def schema(self) -> str | None: ... + @property + def logical_database(self) -> str | None: ... + @property + def logical_schema(self) -> str | None: ... + @property + def fingerprint_database(self) -> str | None: ... + @property + def fingerprint_schema(self) -> str | None: ... + @property + def fingerprint_logical_database(self) -> str | None: ... + @property + def fingerprint_logical_schema(self) -> str | None: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def assembly_row( + self, + ) -> tuple[str, str, list[str], list[tuple[str, str, str | None]]]: ... + +class NativeCompiledFunction: + @property + def key(self) -> CompiledObjectKey: ... + @property + def deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def arguments(self) -> tuple[FunctionArgument, ...]: ... + @property + def returns(self) -> str: ... + @property + def body_sql(self) -> str: ... + @property + def return_columns(self) -> tuple[FunctionReturnColumn, ...]: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def language(self) -> FunctionLanguage: ... + @property + def runtime_version(self) -> str | None: ... + @property + def entry_point(self) -> str | None: ... + @property + def packages(self) -> tuple[str, ...]: ... + @property + def tags(self) -> tuple[str, ...]: ... + @property + def description(self) -> str | None: ... + @property + def relative_path(self) -> Path: ... + @property + def source_file_path(self) -> Path | None: ... + @property + def destination(self) -> CompiledRelationLocation: ... + @property + def fingerprint_destination(self) -> CompiledRelationLocation: ... + def with_target_defaults( + self, + defaults: tuple[str | None, str | None], + render: Callable[..., str | None], + python_functions_inherit_default_namespace: bool, + ) -> NativeCompiledFunction: ... + +class NativeAuditInput: + @property + def audit_file(self) -> DiscoveredAuditFile: ... + @property + def audit_block(self) -> DiscoveredAuditBlock: ... + @property + def sql_body(self) -> str: ... + @property + def evaluation_mode(self) -> AuditEvaluationMode: ... + @property + def measurement_contract(self) -> MeasurementContract | None: ... + @property + def thresholds(self) -> MeasurementThresholds | None: ... + @property + def minimum_samples(self) -> int | None: ... + @property + def measure_sql(self) -> str | None: ... + @property + def evidence_sql(self) -> str | None: ... + @property + def evidence_limit(self) -> int | None: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def attached_target_kind(self) -> AttachedAuditTargetKind | None: ... + @property + def attached_target_name(self) -> str | None: ... + @property + def attached_column_name(self) -> str | None: ... + @property + def severity(self) -> AuditSeverity | None: ... + @property + def run_scope(self) -> str | None: ... + @property + def always_run(self) -> bool: ... + @property + def description(self) -> str | None: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def name(self) -> str | None: ... + @property + def assembly_row( + self, + ) -> tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]: ... + +class NativeCompiledAudit: + @property + def key(self) -> CompiledObjectKey: ... + @property + def scope_deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def definition_name(self) -> str: ... + @property + def audit_file(self) -> DiscoveredAuditFile: ... + @property + def audit_block(self) -> DiscoveredAuditBlock: ... + @property + def sql_body(self) -> str: ... + @property + def evaluation_mode(self) -> AuditEvaluationMode: ... + @property + def measurement_contract(self) -> MeasurementContract | None: ... + @property + def thresholds(self) -> MeasurementThresholds | None: ... + @property + def minimum_samples(self) -> int | None: ... + @property + def measure_sql(self) -> str | None: ... + @property + def evidence_sql(self) -> str | None: ... + @property + def evidence_limit(self) -> int | None: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def attached_target_kind(self) -> AttachedAuditTargetKind | None: ... + @property + def attached_target_name(self) -> str | None: ... + @property + def attached_column_name(self) -> str | None: ... + @property + def severity(self) -> AuditSeverity | None: ... + @property + def run_scope(self) -> str | None: ... + @property + def always_run(self) -> bool: ... + @property + def description(self) -> str | None: ... + +class NativeProjectView: + @property + def run_id(self) -> str: ... + @property + def effective_target_name(self) -> str | None: ... + @property + def effective_connection(self) -> dict[str, object]: ... + @property + def effective_vars(self) -> dict[str, object]: ... + @property + def binding_catalog(self) -> Any | None: ... + @property + def effective_target_database(self) -> str | None: ... + @property + def effective_target_schema(self) -> str | None: ... + @property + def sql_analysis_dialect(self) -> str | None: ... + @property + def sql_lexical_syntax(self) -> SqlLexicalSyntax: ... + @property + def compile_cache_dir(self) -> Path | None: ... + @property + def settings(self) -> SettingsConfig: ... + @property + def scenario(self) -> ScenarioConfig: ... + @property + def models(self) -> tuple[CompiledModelView, ...]: ... + @property + def sources(self) -> tuple[CompiledSource, ...]: ... + @property + def seeds(self) -> tuple[CompiledSeed, ...]: ... + @property + def functions(self) -> tuple[CompiledFunction, ...]: ... + @property + def audits(self) -> tuple[CompiledAudit, ...]: ... + @property + def sql_tests(self) -> tuple[CompiledSqlTest, ...]: ... + @property + def sql_scenarios(self) -> tuple[CompiledSqlScenario, ...]: ... + @property + def loader_functions(self) -> tuple[DiscoveredLoaderFunction, ...]: ... + @property + def hook_functions(self) -> tuple[DiscoveredHookFunction, ...]: ... + @property + def enforce_explicit_references(self) -> bool: ... + @property + def sql_hook_files(self) -> tuple[DiscoveredSqlHookFile, ...]: ... + @property + def materialization_files(self) -> tuple[DiscoveredMaterializationFile, ...]: ... + @property + def public_enums(self) -> dict[str, EnumDeclaration]: ... + @property + def public_constants(self) -> dict[str, ConstantDeclaration]: ... + @property + def loaded_macros(self) -> dict[str, LoadedMacro]: ... + @property + def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... + @property + def unmatched_literal_sql_relations(self) -> tuple[LiteralSqlRelation, ...]: ... + @property + def external_sql_reference_resolver(self) -> ExternalSqlReferenceResolver | None: ... + @property + def scope_index(self) -> ScopeIndex: ... + @property + def sql_expansions(self) -> dict[Path, CompiledSqlExpansion]: ... + @property + def native_session(self) -> Any | None: ... + @property + def native_project(self) -> NativeCompiledProject | None: ... + @property + def lineage_graph(self) -> NativeProjectGraph: ... + def with_changes(self, **changes: object) -> NativeProjectView: ... + def fields(self, include_sql_expansions: bool = True) -> dict[str, Any]: ... + +class NativeCompiledProject: + def __init__(self) -> None: ... + def record_model(self, model: tuple[object, ...]) -> None: ... + def record_model_analysis( + self, + name: str, + columns: tuple[bool, list[tuple[str, str | None]], list[bool], bool, list[str]], + dynamic_proof: tuple[ + bool, + bool, + list[tuple[str, str | None]], + str | None, + list[tuple[str, str | None]], + ] + | None, + ) -> bool: ... + def record_view_analyses(self, views: list[NativeCompiledModel]) -> int: ... + def model_names(self) -> list[str]: ... + def project_view(self, fields: dict[str, object]) -> NativeProjectView: ... + def inputs_view(self, fields: dict[str, object]) -> NativeInputsView: ... + def attach_seeds( + self, + declarations: list[tuple[SchemaSeedEntry, DiscoveredSchemaFile]], + seed_files: list[DiscoveredSeedFile], + compile_error: type[Exception], + ) -> list[NativeSeedInput]: ... + def record_seed_namespaces( + self, namespaces: list[tuple[str | None, str | None, str | None, str | None]] + ) -> None: ... + def compiled_seed(self, index: int) -> NativeCompiledSeed: ... + def attach_sources( + self, + rows: list[tuple[str, DiscoveredSourceFile, SourceEntry]], + settings: tuple[str | None, bool, object], + ) -> list[NativeSourceInput]: ... + def record_source_namespaces( + self, namespaces: list[tuple[str | None, str | None] | None] + ) -> None: ... + def compiled_source(self, index: int) -> NativeCompiledSource: ... + def attach_functions( + self, + rows: tuple[list[tuple[object, ...]], list[tuple[object, ...]]], + settings: tuple[ + str, + bool, + tuple[str | None, str | None], + tuple[str | None, str | None] | None, + bool, + object, + ], + ) -> list[NativeFunctionInput]: ... + def record_function_deps(self, deps: list[list[tuple[str, str]]]) -> None: ... + def compiled_function(self, index: int) -> NativeCompiledFunction: ... + def resource_diagnostics( + self, settings: dict[str, object] + ) -> tuple[list[CompilerDiagnostic], list[CompilerDiagnostic], list[CompilerDiagnostic]]: ... + def attach_sql_tests( + self, + files: list[tuple[object, ...]], + scenarios: list[tuple[object, ...]], + settings: dict[str, object], + ) -> tuple[list[CompileSqlTestInput], list[CompileSqlScenarioInput]]: ... + def attach_audits( + self, files: list[tuple[object, ...]], settings: dict[str, object] + ) -> tuple[list[NativeAuditInput], list[CompilerDiagnostic]]: ... + def record_audit_scope_deps(self, deps: list[list[tuple[str, str]]]) -> None: ... + def compiled_audit(self, index: int) -> NativeCompiledAudit: ... + def graph_builds(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + +class NativeMacroExpansion: + def __init__( + self, + file_path: object, + state: tuple[object, object, object, object, object], + enforce_explicit: bool, + classes: dict[str, object], + ) -> None: ... + def expand( + self, sql: str, bridge: object + ) -> tuple[ + str, + list[tuple[int, int, int, int]], + list[tuple[str, str]], + list[Any], + list[Any], + ]: ... + def render_relation(self, reference: object) -> str: ... + def declaration_read(self, kind: str, name: str) -> None: ... + def block_memo(self) -> None: ... + +class NativeModelLoop: + def __init__( + self, + checks: tuple[object, dict[str, object], dict[str, object]], + syntax: dict[str, object], + cursor: tuple[list[str], tuple[tuple[int, int], str]], + settings: tuple[tuple[bool, bool, bool, bool], tuple[str, list[tuple[str, list[str]]]]], + ) -> None: ... + def run( + self, + project: NativeCompiledProject, + host: object, + models: list[tuple[object, ...]], + ) -> tuple[list[NativeModelInput], tuple[int, int, int, int, int]]: ... + +def relation_key_row( + kind: str, name: str, values: dict[str, object], entry: object +) -> RelationKeyRow: ... +def declared_relation_keys(rows: list[RelationKeyRow]) -> dict[str, list[list[str]]]: ... + +class NativeProjectGraph: + @staticmethod + def from_resources( + resources: list[tuple[_GraphKey, list[_GraphKey], list[str], str | None]], + ) -> NativeProjectGraph: ... + @staticmethod + def from_indexes(indexes: _GraphIndexes) -> NativeProjectGraph: ... + def indexes(self) -> _GraphIndexes: ... + def upstream(self, key: _GraphKey) -> list[_GraphKey]: ... + def downstream(self, key: _GraphKey) -> list[_GraphKey]: ... + def keys(self) -> list[_GraphKey]: ... + def names(self) -> list[tuple[str, _GraphKey]]: ... + def edges(self) -> list[tuple[_GraphKey, _GraphKey]]: ... + def model_layer_count(self) -> int: ... + def unique_id_maps( + self, project_name: str, prefixes: dict[str, str] + ) -> tuple[list[tuple[str, list[str]]], list[tuple[str, list[str]]]]: ... + def resolve(self, select: list[str], exclude: list[str]) -> _SelectorKeys: ... + def tokens(self, selectors: list[str]) -> _SelectorKeys: ... + def matched(self, kind: str, value: str) -> _SelectorKeys: ... + def build_resources( + self, selected: list[_GraphKey], include: tuple[bool, bool, bool] + ) -> list[_GraphKey]: ... + +def parse_project_selector( + raw: str, +) -> tuple[tuple[str, str, str, bool, bool] | None, _SelectorFailure | None]: ... +def project_selector_name_help(value: str, candidates: list[str]) -> str | None: ... + +class NativeReuseAttempt: + def outcome(self) -> str: ... + def store_path(self) -> str: ... + def replay_failed(self) -> bool: ... + def snapshot_paths(self) -> int: ... + def pending_digest_bytes(self) -> int: ... + def replay(self, timings: list[tuple[str, int]]) -> tuple[str, list[str], int] | None: ... + def digested_files(self) -> int: ... + def launch_identity(self) -> tuple[str, dict[str, str], list[str]]: ... + def record(self, record: dict[str, object]) -> tuple[str, int]: ... + +def check_compile_reuse(check: dict[str, object]) -> NativeReuseAttempt: ... +def artifact_digest(contents: bytes) -> str: ... +def write_compiled_artifacts( + files: list[tuple[Path, bytes]], check_existing: bool +) -> tuple[int, int]: ... +def staged_artifact_files(staged_dir: Path) -> list[str]: ... +def remove_stale_artifacts(compiled_dir: Path, managed: list[str]) -> int: ... +def publish_staged_artifacts( + staged_dir: Path, compiled_dir: Path, expected: list[str] +) -> tuple[str, list[tuple[str, str]]]: ... +def finalize_rule_findings_rows( + request: dict[str, object], +) -> list[tuple[bool, str, str, int, int, str, str]]: ... def run_custom_host_json(spec_json: str) -> str: ... def format_sql_json(request_json: str) -> str: ... def format_sql_batch_json(request_json: str) -> str: ... @@ -54,6 +822,15 @@ def parse_model_headers( headers: list[str], ) -> list[tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None]]: ... def tokenize_model_header(header: str) -> list[tuple[int, str, int]]: ... +def model_output_columns( + contents: str, implicit_aliases: bool +) -> list[tuple[str, int, int, int, int]]: ... +def model_header_column_locations( + contents: str, header: str, header_start: int +) -> tuple[list[tuple[str, int, int, int, int]] | None, str | None]: ... +def model_header_failure( + error: str, header: str, header_line: int | None +) -> tuple[str, str | None, int | None]: ... def native_text_supported(python_version: tuple[int, int], unicode_version: str) -> bool: ... class NativeProjectTree: @@ -103,9 +880,11 @@ class NativeDeclarationContexts: positions: list[tuple[object, ...]], table: tuple[list[int], dict[str, list[int]], dict[str, list[int]]], classes: dict[str, object], - lookup: dict[str, object], + lookups: tuple[dict[str, object], dict[str, object]], ) -> None: ... - def context(self, matches: Sequence[Any], consumer: object) -> Any: ... + def resolve( + self, resolver: object, file_path: object, resource: object | None = None + ) -> tuple[Any, bool]: ... class NativeScopeIndex: def records( @@ -119,24 +898,59 @@ class NativeScopeIndex: list[tuple[str, str, str | None, int | None, int | None, int | None, int | None]], bool, ]: ... - def grant( - self, facts: list[tuple[str, str, list[str], list[str], list[str]]] - ) -> list[tuple[str, str, int, str | int, str]] | None: ... - def lookup( + def grant_relationships( self, - ) -> ( + files: list[tuple[str, list[tuple[str, str, str]]]], + scenarios: list[tuple[str, str, str]], + settings: dict[str, object], + ) -> tuple[list[tuple[str, str, int, str | int, str]] | None, list[tuple[str, str]]]: ... + +def scope_placement_diagnostics( + rows: tuple[list[Any], list[Any], list[Any]], +) -> list[tuple[int, str, str]]: ... +def scope_relocations( + rows: tuple[list[Any], list[Any], list[Any]], + resource: tuple[str, str], + destination: str, +) -> list[tuple[int, str, str, tuple[str, str, str | None], str | None]] | None: ... +def scope_declaration_directory(kind: str, scope: str, path: str | None) -> str: ... + +type _ScopeQualified = tuple[str, str, tuple[str, str] | None] +type _ScopeRoot = tuple[str, str, str | None] +type _ScopeResource = tuple[tuple[str, str], str, _ScopeRoot] +type _ScopeGroups = list[list[int]] + +class NativeScopeLookup: + def target( + self, identity: _ScopeQualified | None, text: str, / + ) -> tuple[bool, list[int], str | None, str | None]: ... + def resolve( + self, resource: _ScopeResource, / + ) -> tuple[list[tuple[int, str, int | None]], list[tuple[int, str]], str | None]: ... + +def build_native_scope_lookup( + rows: tuple[ + list[_ScopeRoot], + list[_ScopeResource], + list[tuple[_ScopeQualified, str, int, int, str, _ScopeRoot, str | None]], + list[tuple[_ScopeQualified, _ScopeQualified, str, _ScopeQualified | None]], + list[tuple[tuple[str, str], _ScopeQualified, _ScopeQualified, str]], + list[tuple[str | None, int | None, int | None, str]], + ], + /, +) -> tuple[ + NativeScopeLookup | None, + tuple[ tuple[ - list[int], - list[int], - list[int], - list[int], + tuple[list[int], list[int], list[int], list[int], list[int], list[int]], tuple[ - tuple[list[list[int]], bool], - tuple[list[list[int]], bool], - tuple[list[list[int]], bool], - tuple[list[list[int]], bool], - tuple[list[list[int]], bool], - tuple[list[list[int]], bool], + _ScopeGroups, + _ScopeGroups, + _ScopeGroups, + _ScopeGroups, + _ScopeGroups, + _ScopeGroups, + list[str], ], tuple[ list[int], @@ -145,9 +959,128 @@ class NativeScopeIndex: list[tuple[str, list[int]]], ], ] - | None - ): ... + | None, + str | None, + ], +]: ... +class NativeScopeReports: + def report(self, request: tuple[Any, ...], /) -> tuple[Any, tuple[str, str] | None]: ... + def explain(self, request: tuple[Any, ...], /) -> tuple[Any, tuple[str, str] | None]: ... + def browse(self, request: tuple[Any, ...], /) -> Any: ... + def list(self, request: tuple[Any, ...], /) -> tuple[Any, tuple[str, str] | None]: ... + def move_preview(self, request: tuple[Any, ...], /) -> tuple[Any, tuple[str, str] | None]: ... + +def build_native_scope_reports( + lookup: NativeScopeLookup, + values: tuple[list[Any], tuple[bool, bool, bool, bool, bool, bool]], + /, +) -> NativeScopeReports: ... +def serialize_scope_result_row(kind: str, row: object, /) -> str: ... +def scope_metadata_projection_rows(rows: tuple[Any, ...], /) -> Any: ... +def effective_config_target_name( + project_config: object, local_config: object, selected: str | None, / +) -> tuple[str | None, str | None]: ... +def effective_config_target_fields( + configs: tuple[object, object], target_name: str, default_target: object, / +) -> tuple[list[tuple[str | None, str, object]] | None, str | None]: ... +def effective_config_vars( + configs: tuple[object, object], target_config: object | None, cli_vars: dict[str, object], / +) -> dict[str, object]: ... +def effective_config_connection( + project_config: object, local_config: object, target_config: object | None, / +) -> tuple[dict[str, object] | None, str | None]: ... +def effective_config_settings( + settings: tuple[object, object], overrides: frozenset[str], names: list[str], / +) -> dict[str, object]: ... +def yaml_entry_line_index(contents: str, /) -> list[tuple[str, int]]: ... + +class NamePattern: + def __init__(self, pattern: str) -> None: ... + def matches(self, column_name: str) -> bool: ... + +def unresolved_reference_marker_kind( + sql: str, context: str, / +) -> tuple[int | None, str | None]: ... +def normalize_typed_sql_value( + raw_value: object, typing: tuple[str, str | None], limits: tuple[int, int, int], / +) -> object: ... +def sql_test_blocks( + file_path: str, blocks: list[tuple[dict[str, object], str, tuple[int, int]]], / +) -> list[object]: ... +def validate_sql_test_block_names( + file_path: str, blocks: list[tuple[int, str | None]], / +) -> None: ... +def model_header_declarations( + header: tuple[object | None, object | None], model_name: str, relative_path: Path, / +) -> tuple[tuple[object, ...], tuple[object, ...]]: ... +def typed_constant_record( + constant: tuple[str, object], + options: tuple[str | None, str | None], + paths: tuple[Path, Path], + /, +) -> object: ... +def deferred_target_segments( + template: str, logical: str, context_keys: list[str], variables: list[str], / +) -> list[tuple[bool, str]]: ... +def unknown_mapping_keys( + keys: list[tuple[str, bool]], allowed: list[str], / +) -> tuple[str, str] | None: ... +def unsupported_keys_help(keys: list[str], supported_keys: list[str], /) -> str: ... +def resource_sql_diagnostic_rows(catalog: object, request: object, /) -> list[object]: ... +def compiler_query_hash(sql: str, dialect: str | None, /) -> str: ... +def function_definition_hash(sql: str, language: str, dialect: str | None, /) -> str: ... +def normalize_query_text(sql: str, /) -> str: ... +def glob_name_match(name: str, pattern: str, ignore_case: bool, /) -> bool: ... +def lone_surrogate_site(raw_values: dict[str, object], /) -> tuple[str, str, int] | None: ... +def environment_template_names(values: list[str | None], /) -> list[str]: ... +def contract_model_edit( + row: tuple[ + str, + str, + bool, + list[tuple[str, str | None, str | None]], + list[str | None], + list[tuple[str, str]], + bool, + ], + /, +) -> tuple[str | None, str | None, str | None]: ... +def contract_source_edit( + row: tuple[str, str, str, list[tuple[str, str | None]], list[tuple[str, str]], bool], / +) -> tuple[str | None, str | None, str | None]: ... +def resource_identity_suggestion( + name: str, private_identity: bool, / +) -> tuple[str, str] | None: ... +def load_project_config_file(project_dir: Path, /) -> object: ... +def load_local_config_file(project_dir: Path, /) -> object: ... +def resolve_config_file_path(project_dir: Path, local: bool, /) -> Path: ... +def validate_local_sql_analysis(project_dir: Path, flags: tuple[bool, bool, bool], /) -> None: ... +def sql_test_cte_span( + contents: str, sql_body: str, cte_name: str, call: str | None, / +) -> tuple[int, int, int, int]: ... +def scope_normalize_path(path: str, /) -> tuple[str | None, str | None]: ... +def scope_index_fingerprint( + project_dir: Path, + inputs: tuple[list[str], list[str], list[str], list[str], str], + /, +) -> str | None: ... +def read_scope_index_cache(path: Path, fingerprint: str, /) -> Any: ... +def write_scope_index_cache(location: tuple[Path, str], fingerprint: str, rows: Any, /) -> bool: ... +def scope_schema_usages( + schemas: list[tuple[str, str | None, list[tuple[str, str | None]]]], + records: list[tuple[tuple[str, str], tuple[str, str, tuple[str, str] | None, str | None, str]]], +) -> tuple[list[tuple[int, int]], list[tuple[list[str], str, str, str]], str | None]: ... +def scope_merged_usages( + declarations: list[ + tuple[ + tuple[str, str, tuple[str, str] | None], + list[tuple[str, str, tuple[str, str] | None]] | None, + ] + ], + prior: list[Any], + collected: list[Any], +) -> tuple[list[int], list[list[tuple[bool, int]] | None], list[int]]: ... def build_native_scope_index( resources: list[tuple[str, str, str, str]], declarations: list[ @@ -173,11 +1106,15 @@ class NativeConfigError: help: str | None key: str | None -def parse_model_header_metadata( - header: tuple[object, object], - source: tuple[dict[str, Any], str], - classes: dict[str, object], -) -> tuple[tuple[Any, ...] | NativeConfigError, tuple[Any, ...] | NativeConfigError | None]: ... +def column_migration_problem( + model_name: str, values: dict[str, object], columns: list[tuple[str, str | None]] +) -> NativeConfigError | None: ... +def expand_declaration_references( + source: tuple[str, list[tuple[int, str, str | None, int, int]], int | None], + context: object, + consumer: tuple[object | None, bool, str], + renderers: tuple[object, dict[str, object]], +) -> tuple[str, list[tuple[int, int, int, int]], list[Any]]: ... def expand_config_templates( value: object, sources: tuple[dict[str, object], object, dict[str, str | None]], @@ -195,19 +1132,12 @@ class NativeModelConfigBuilder: target: tuple[tuple[str | None, str], tuple[str | None, str | None] | None], python: tuple[tuple[int, int], str], ) -> None: ... - def path_default(self, model_path: str) -> str | NativeConfigError | None: ... - def build( - self, header: dict[str, object], matched_path_default: str | None, model_name: str - ) -> ( - tuple[ - dict[str, object], - tuple[str, ...], - tuple[str | None, bool, str | None], - tuple[tuple[int | None, bool] | None, str | None], - list[tuple[str, str]], - ] - | NativeConfigError - ): ... + def model_config( + self, + header: dict[str, object], + model: tuple[str | None, str], + classes: dict[str, object], + ) -> object | NativeConfigError: ... class NativeModelValidator: def __init__( @@ -248,32 +1178,8 @@ def scope_expected_model_names( def scope_test_ctes( texts: list[tuple[str, str]], syntax: dict[str, object] ) -> list[tuple[str | None, list[tuple[str, str]]]]: ... -def extract_sql_scenario_json(sql: str, file_label: str, syntax: dict[str, object]) -> str: ... - -class SqlTestTargetCatalog: - def __init__( - self, - models: set[str], - sources: set[str], - seeds: set[str], - resources: tuple[set[str], set[str]], - ) -> None: ... - def unknown_test_target( - self, - file_label: str, - targets: tuple[list[str], list[str], list[str], list[str], list[str], list[str], list[str]], - ) -> str | None: ... - def scenario_source_error( - self, file_label: str, ctes: list[tuple[str, bool, list[str]]] - ) -> str | None: ... - def omitted_ceremonial_select(sql: str, syntax: dict[str, object]) -> int | None: ... -def scan_test_parameter_references( - sql: str, declared: list[str], owner: str -) -> tuple[list[tuple[int, int, str]], str | None]: ... -def cursor_intrinsics_rejection( - sql: str, reserved_markers: list[str], context: str, python: tuple[tuple[int, int], str] -) -> str | None: ... +def extract_sql_tests_json(request_json: str) -> str: ... def validated_model_cursor_intrinsics( sql: str, reserved_markers: list[str], @@ -283,29 +1189,6 @@ def validated_model_cursor_intrinsics( def replace_cursor_intrinsics( sql: str, context: str, replacements: tuple[str, str], python: tuple[tuple[int, int], str] ) -> tuple[str, bool, str | None]: ... -def parse_function_header_values( - header_values: dict[str, object], python: bool, relative_path: str -) -> tuple[ - list[tuple[str, str, str]], - str | None, - list[tuple[str, str, str]] | None, - list[str], - str | None, - str | None, - str | None, - list[str], - tuple[str, str] | None, -]: ... -def resolve_function_namespace_values(inputs: dict[str, object]) -> list[str | None]: ... -def pair_seed_files( - declarations: list[str], stems: list[str] -) -> tuple[list[int], None] | tuple[None, int]: ... -def render_attached_generic_audit( - labels: tuple[str, str], - sql: tuple[str, str | None], - arguments: tuple[list[tuple[str, str]], dict[str, object]], - policies: dict[str, object], -) -> tuple[str | None, str, str | None, str, str, str | None]: ... def parse_macro_arguments(text: str, nested: list[tuple[int, int]]) -> tuple[object, ...]: ... def scan_sql_declaration_references( sqls: list[str], python_version: tuple[int, int], unicode_version: str @@ -332,7 +1215,6 @@ def scan_macro_call_sites( list[tuple[int, int, int, str, list[str] | None, bool]], tuple[int | None, str | None, str] | None, ]: ... -def sql_test_helper_reads(sql: str, keys: list[str], /) -> list[str] | None: ... def extract_sql_relation_names( sql: str, dialect: str | None ) -> tuple[list[tuple[str, str | None, str | None]], list[str]] | None: ... @@ -347,29 +1229,27 @@ def splice_macro_calls( class MacroCallMemo: def __init__(self) -> None: ... - def lookup( - self, class_id: int, call_text: str - ) -> tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]] | None: ... - def record( - self, - class_id: int, - call_text: str, - entry: tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]], - ) -> None: ... + def reset_call_classes(self, store_tokens: bool) -> None: ... + def observe_modules(self, count: int | None) -> None: ... def stats(self) -> tuple[int, int, int]: ... def attach_store(self, path: str, environment: str) -> tuple[bytes, int]: ... def discard_store(self) -> None: ... - def set_persistent_class(self, class_id: int, class_text: str) -> None: ... def store_stats(self) -> tuple[int, int]: ... def save_store(self, path: str, metadata: bytes) -> int | None: ... class SqlTestScanStore: - def __init__(self, path: str, environment: str) -> None: ... - def get(self, key_parts: list[str]) -> bytes | None: ... - def put(self, key_parts: list[str], value: bytes) -> None: ... - def save(self, path: str) -> int | None: ... + @staticmethod + def open_in( + cache_dir: str, code_identity: str + ) -> tuple[SqlTestScanStore | None, str | None]: ... + def counts(self) -> tuple[int, int]: ... + def save(self) -> tuple[int | None, str | None]: ... + +SQL_TEST_SCAN_STORE_FILE_NAME: str def content_digest(parts: list[str]) -> str: ... +def installed_code_identity(facts: list[str], package_root: str | None) -> str: ... +def compiled_code_identity(installed: str, build_identity: str) -> str: ... def fingerprint_project_files(root: str, excluded_files: list[str]) -> str: ... def digest_files(paths: list[str]) -> list[str | None]: ... @@ -394,6 +1274,14 @@ type _PivotContractRow = tuple[ | None, ] +def locate_model_binding_diagnostics( + rows: list[_AnalysisDiagnosticRow], + texts: tuple[str, str, str], + expansion: tuple[str, list[list[tuple[int, int, int, int]]]] | None, +) -> list[ + tuple[str, str, str, int | None, int | None, tuple[int, int, int | None, int | None] | None] +]: ... + class NativeModelAnalysisSession: @property def fact_models(self) -> list[str]: ... @@ -402,37 +1290,9 @@ class NativeModelAnalysisSession: ) -> tuple[list[tuple[str, list[tuple[str, str]]]], list[str]]: ... def finish( self, - ) -> tuple[ - list[ - tuple[ - bool, - list[tuple[str, str | None, str]] | None, - str, - list[tuple[str, int, int, list[tuple[str, str, str]]]], - bool, - bool, - list[_AnalysisDiagnosticRow], - bool, - str, - ] - ], - list[tuple[str, list[tuple[str, str]]]], - list[str], - list[ - tuple[ - str, - tuple[ - bool, - list[tuple[str, str | None, str]], - list[tuple[str, str | None]], - list[str], - str | None, - bool, - ] - | None, - ] - ], - ]: ... + ) -> tuple[list[tuple[str, list[tuple[str, str]]]], list[str], list[bool]]: ... + def binding_rows(self) -> list[list[_AnalysisDiagnosticRow]]: ... + def outcome_rows(self, index: int) -> tuple[Any, Any]: ... def prove_dynamic_contracts( self, models: list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], / ) -> list[_PivotContractRow]: ... @@ -476,6 +1336,7 @@ def infer_expression_source_shapes( request: tuple[str, bool, list[tuple[str, str]], list[tuple[str, str]], list[str]], adapter_rules: tuple[Mapping[str, Callable[..., object]], type] | None = None, ) -> list[list[tuple[str, str]] | None]: ... +def _oracle_names_set_operation(sql: str, /) -> bool: ... def _oracle_cte_fact_recovery( request: tuple[ str, @@ -525,29 +1386,24 @@ def check_semantic_metadata_rows( catalog: object, request: tuple[ str | None, + bool, list[tuple[str, str]], list[tuple[str, list[tuple[str, str]]]], list[tuple[str, list[tuple[str, str]]]], - list[ - tuple[ - str, - str, - str, - bool, - bool, - list[tuple[str, list[str]]], - list[tuple[str, list[str]]], - str | None, - str | None, - ] - ], + list[tuple[object, ...]], list[str], list[tuple[str, str | None, int]], list[tuple[int, list[tuple[str, list[str]]]]], ], /, ) -> tuple[ - list[tuple[list[tuple[str, str, int, int]], list[tuple[str, str, int, int]]]], + list[ + tuple[ + list[tuple[str, str, int, int]], + list[tuple[str, str, int, int, bool]], + list[tuple[str, str, int, int]], + ] + ], list[tuple[int, tuple[str, str, int, int]]], list[tuple[int, tuple[str, str, int, int], int]], list[str], @@ -603,49 +1459,46 @@ def complete_semantic_checks( ]: ... # Native analysis: contracts. -def evaluate_native_model_contracts( - request: tuple[ +type _ContractDiagnosticRow = tuple[ + str, bool, str, str | None, int | None, str | None, tuple[str, str] | None, str +] + +def evaluate_project_model_contracts( + settings: tuple[str, bool], project: NativeCompiledProject +) -> list[ + tuple[ str, - bool, list[ - tuple[ - str, - str | None, - tuple[list[tuple[str, str | None, bool, bool]], list[tuple[str, str]], bool, bool] - | None, - list[tuple[str, str | None, bool]] | None, - bool, - tuple[bool, str | None, list[tuple[str, str | None]]] | None, - list[str], - ] + tuple[str, bool, str, str | None, int | None, str | None, tuple[str, str] | None, str] ], - ], - /, -) -> list[ - list[ - tuple[ - str, - bool, - str, - str | None, - int | None, - str | None, - tuple[str, str] | None, - str, - ] ] ]: ... -def native_promotion_conflicts( - request: tuple[str | None, str, str, list[tuple[str, str | None, str | None, str | None]]], - /, -) -> list[tuple[int, str, str, str]]: ... - -# Native analysis: column lineage facts. +def project_promotion_conflicts( + settings: tuple[str | None, str, str], project: NativeCompiledProject +) -> list[tuple[str, str, str, str, str]]: ... def build_fast_column_lineage( catalog: object | None, request: tuple[str | None, list[tuple[str, str, list[str]]], list[tuple[bool, str, list[str]]]], /, ) -> list[tuple[str, list[tuple[str, str, str, list[tuple[str, str, str]]]], bool, str | None]]: ... +def build_direct_semantic_uses( + schema: list[tuple[str, str, list[tuple[str, str | None]], list[tuple[str, str | None]]]], + models: list[tuple[str, str, str]], + dialect: str, +) -> list[tuple[str, str, str, str, str, str, str, int | None, int | None]]: ... +def build_rich_column_lineage( + request: tuple[ + str, + list[tuple[str, str, list[tuple[str, str | None]], list[tuple[str, str | None]]]], + list[str], + ], + /, +) -> list[ + tuple[str, list[tuple[str, str, str, str, list[tuple[str, str, str]]]], bool, str | None] +]: ... +def relation_lineage_fingerprint( + project_dir: Path, prefix: bytes, interrupted_listing_uncacheable: bool, / +) -> str | None: ... # Native analysis: SQL test planning glue. def plan_compiled_sql_tests( @@ -705,7 +1558,6 @@ def _oracle_text_positions( ) -> tuple[str, list[tuple[int, int, int] | None]]: ... def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... -def _oracle_project_config(project_dir: str) -> str: ... def _oracle_python_alnum( python_version: tuple[int, int], unicode_version: str, code_points: list[int] ) -> list[bool] | None: ... @@ -715,4 +1567,3 @@ def _oracle_cleandoc( def _oracle_close_matches( word: str, possibilities: list[str], count: int, cutoff: float ) -> list[str]: ... -def function_type_error(type_sql: str, adapter_name: str, context: str) -> str | None: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index 7a54795a27..59bab771b1 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -24,11 +24,13 @@ ) from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, - CompiledModel, CompiledObjectKey, - CompiledProject, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.clone_with_options import run_clone_pipeline_with_options from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project @@ -78,10 +80,10 @@ def build_defer_clone_boundary_selectors( cli_vars: dict[str, object] | None, project_dir: Path, auto_load_sources: bool, -) -> tuple[CompiledProject, tuple[str, ...], tuple[str, ...], int]: +) -> tuple[CompiledProjectView, tuple[str, ...], tuple[str, ...], int]: """Resolve out-of-selection clone boundaries and the view chain to rebuild over them.""" - project: CompiledProject = build_compiled_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, selected_target=selected_target, @@ -110,7 +112,7 @@ def build_defer_clone_boundary_selectors( def _scope_is_recreated(*, scope: PlannerScope, key: CompiledObjectKey) -> bool: if key.resource_type in {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN}: return True - model: CompiledModel | None = scope.models_by_name.get(key.name) + model: CompiledModelView | None = scope.models_by_name.get(key.name) if model is None: return False return ( @@ -159,7 +161,7 @@ def selected_executable_model_count(*, scope: PlannerScope) -> int: for key in scope.selected_keys: if key.resource_type != CompiledResourceType.MODEL: continue - model: CompiledModel | None = scope.models_by_name.get(key.name) + model: CompiledModelView | None = scope.models_by_name.get(key.name) if model is not None and model.config.values.get("enabled") is False: continue count += 1 @@ -185,7 +187,7 @@ def run_defer_clone_boundary_prephase( boundary_selectors=(), view_chain_selectors=(), ) - cloned_project: CompiledProject + cloned_project: CompiledProjectView boundary_selectors: tuple[str, ...] view_chain_selectors: tuple[str, ...] ( diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index e0c6aa3c00..bf98636484 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -2,13 +2,11 @@ from __future__ import annotations -import json from collections import Counter from importlib.metadata import PackageNotFoundError, version from pathlib import Path -import orjson - +import sqlbuild._native as _native from sqlbuild.cli.commands._helpers.compile.semantic_notice import ( selected_semantic_coverage, semantic_coverage_notice, @@ -19,26 +17,28 @@ WrittenTarget, ) from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledFunction, - CompiledModel, CompiledModelSqlTestPayload, CompiledObjectKey, - CompiledProject, - CompiledSeed, - CompiledSource, CompiledSqlTest, CompilerDiagnostic, RelatedLocation, ) from sqlbuild.compiler.compile.types import ( + CompiledAuditView, + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, CompiledResourceType, + CompiledSeedView, + CompiledSourceView, DiagnosticPhase, DiagnosticSeverity, ) from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities @@ -71,7 +71,7 @@ def format_compile_text( """Format human-readable compile output.""" style: CliStyle = CliStyle(use_color=use_color) - visible_project_models: tuple[CompiledModel, ...] = _selected_models( + visible_project_models: tuple[CompiledModelView, ...] = _selected_models( graph=graph, selected_keys=selected_keys, ) @@ -83,7 +83,7 @@ def format_compile_text( ), "", ] - visible_models: tuple[CompiledModel, ...] = visible_project_models[:_HUMAN_MODEL_LIMIT] + visible_models: tuple[CompiledModelView, ...] = visible_project_models[:_HUMAN_MODEL_LIMIT] model_name_width: int = _model_name_width(visible_models) error_models: frozenset[str] = _models_with_error_diagnostics(diagnostics) model_index: int @@ -94,7 +94,7 @@ def format_compile_text( lines.append( f"{connector} {style.object_name(model_name)} " f"{format_status_cell(style=style, status=status, width=4)} " - f"{style.muted(f'{_column_count(model)} columns')}" + f"{style.muted(f'{model.output_column_count} columns')}" ) hidden_model_count: int = len(visible_project_models) - len(visible_models) if hidden_model_count > 0: @@ -214,10 +214,9 @@ def format_compile_json( "resources": _resources(graph=graph, lineage=lineage), "artifacts": _artifacts(written=written, manifest=manifest), } - try: - return orjson.dumps(result, option=orjson.OPT_INDENT_2).decode() - except TypeError: - return json.dumps(result, indent=2) + emitted: str = _native.emit_json_report(result) + report_native_answer(stage=NativeStage.COMPILE_OUTPUTS, kind="json_reports") + return emitted def format_compile_error_json( @@ -244,7 +243,9 @@ def format_compile_error_json( "stopped": True, "diagnostics": [_diagnostic_to_json(diagnostic)], } - return orjson.dumps(result, option=orjson.OPT_INDENT_2).decode() + emitted: str = _native.emit_orjson_report(result) + report_native_answer(stage=NativeStage.COMPILE_OUTPUTS, kind="json_reports") + return emitted def _summary( @@ -253,7 +254,7 @@ def _summary( diagnostics: tuple[CompilerDiagnostic, ...], selected_keys: frozenset[CompiledObjectKey] | None = None, ) -> dict[str, int]: - project: CompiledProject = graph.project + project: CompiledProjectView = graph.project return { "models": len(project.models), "selected_models": len(_selected_models(graph=graph, selected_keys=selected_keys)), @@ -283,7 +284,7 @@ def _selected_models( *, graph: ProjectGraph, selected_keys: frozenset[CompiledObjectKey] | None, -) -> tuple[CompiledModel, ...]: +) -> tuple[CompiledModelView, ...]: if selected_keys is None: return graph.project.models selected_names: frozenset[str] = frozenset( @@ -298,14 +299,14 @@ def _selected_resource_count( selected_keys: frozenset[CompiledObjectKey] | None, resource_types: set[CompiledResourceType], ) -> int: - effective_keys: frozenset[CompiledObjectKey] = ( - frozenset(graph.all_keys.values()) if selected_keys is None else selected_keys - ) - return sum(key.resource_type in resource_types for key in effective_keys) + if selected_keys is None: + type_names: frozenset[str] = frozenset(map(str, resource_types)) + return sum(kind in type_names for kind, _ in graph.native.keys()) + return sum(key.resource_type in resource_types for key in selected_keys) def _resources(*, graph: ProjectGraph, lineage: ProjectColumnLineage | None) -> dict[str, object]: - project: CompiledProject = graph.project + project: CompiledProjectView = graph.project return { "models": [ _model_resource(graph=graph, model=model, lineage=lineage) for model in project.models @@ -322,15 +323,18 @@ def _resources(*, graph: ProjectGraph, lineage: ProjectColumnLineage | None) -> def _model_resource( *, graph: ProjectGraph, - model: CompiledModel, + model: CompiledModelView, lineage: ProjectColumnLineage | None, ) -> dict[str, object]: item: dict[str, object] = { "name": model.name, "relative_path": str(model.relative_path), "materialized": str(model.config.values.get("materialized", "view")), - "column_count": _column_count(model), - "depends_on": [_serialize_key(dep) for dep in graph.upstream_deps.get(model.key, ())], + "column_count": model.output_column_count, + "depends_on": [ + {"resource_type": kind, "name": name} + for kind, name in graph.native.upstream((str(model.key.resource_type), model.key.name)) + ], "lineage": _lineage_summary(lineage=lineage, model=model), "query_sql": model.query_sql, } @@ -347,7 +351,7 @@ def _hook_resources(*, hooks: object) -> list[dict[str, object]]: return serialize_hook_entries(value=hooks, sql_fields=SQL_HOOK_OUTPUT_FIELDS) -def _hook_definitions(project: CompiledProject) -> list[dict[str, object]]: +def _hook_definitions(project: CompiledProjectView) -> list[dict[str, object]]: definitions: list[dict[str, object]] = [ { "type": "sql", @@ -372,7 +376,7 @@ def _hook_definitions(project: CompiledProject) -> list[dict[str, object]]: return definitions -def _source_resource(source: CompiledSource) -> dict[str, object]: +def _source_resource(source: CompiledSourceView) -> dict[str, object]: return { "name": source.name, "relative_path": str(source.source_file.relative_path), @@ -380,7 +384,7 @@ def _source_resource(source: CompiledSource) -> dict[str, object]: } -def _seed_resource(seed: CompiledSeed) -> dict[str, object]: +def _seed_resource(seed: CompiledSeedView) -> dict[str, object]: item: dict[str, object] = { "name": seed.name, "relative_path": str(seed.seed_file.relative_path), @@ -391,7 +395,7 @@ def _seed_resource(seed: CompiledSeed) -> dict[str, object]: return item -def _function_resource(function: CompiledFunction) -> dict[str, object]: +def _function_resource(function: CompiledFunctionView) -> dict[str, object]: item: dict[str, object] = { "name": function.name, "relative_path": str(function.relative_path), @@ -408,7 +412,7 @@ def _function_resource(function: CompiledFunction) -> dict[str, object]: return item -def _audit_resource(audit: CompiledAudit) -> dict[str, object]: +def _audit_resource(audit: CompiledAuditView) -> dict[str, object]: item: dict[str, object] = { "name": audit.name, "relative_path": str(audit.audit_file.relative_path), @@ -468,7 +472,7 @@ def _test_resource(test: CompiledSqlTest) -> dict[str, object]: def _lineage_summary( - *, lineage: ProjectColumnLineage | None, model: CompiledModel + *, lineage: ProjectColumnLineage | None, model: CompiledModelView ) -> dict[str, object]: if lineage is None: return {"available": False} @@ -476,7 +480,7 @@ def _lineage_summary( return {"available": False} return { "available": True, - "column_count": _column_count(model), + "column_count": model.output_column_count, "edge_count": lineage.edge_count_targeting(model.name), "has_star": lineage.model_has_star(model.name), } @@ -489,15 +493,7 @@ def _artifacts(*, written: WrittenTarget, manifest: bool) -> dict[str, object]: } -def _column_count(model: CompiledModel) -> int: - if model.inferred_columns is not None: - return len(model.inferred_columns) - if model.schema_entry is not None: - return len(model.schema_entry.columns) - return 0 - - -def _model_name_width(models: tuple[CompiledModel, ...]) -> int: +def _model_name_width(models: tuple[CompiledModelView, ...]) -> int: longest: int = max((len(model.name) for model in models), default=0) return min(max(longest, _MIN_MODEL_NAME_WIDTH), _MAX_MODEL_NAME_WIDTH) @@ -701,7 +697,7 @@ def _caret_line(*, location: SourceLocation, source_line: str, message: str | No return " " * (start_column - 1) + "^" * caret_count + suffix -def _model_source_texts(models: tuple[CompiledModel, ...]) -> dict[Path, str]: +def _model_source_texts(models: tuple[CompiledModelView, ...]) -> dict[Path, str]: return {model.relative_path: model.authored_sql for model in models if model.authored_sql} @@ -713,47 +709,8 @@ def _style_diagnostic(*, text: str, diagnostic: CompilerDiagnostic, style: CliSt return style.muted(text) -def _serialize_key(key: CompiledObjectKey) -> dict[str, str]: - return {"resource_type": str(key.resource_type), "name": key.name} - - def _execution_layer_count(graph: ProjectGraph) -> int: - model_keys: set[CompiledObjectKey] = {model.key for model in graph.project.models} - if not model_keys: - return 0 - remaining_model_deps: dict[CompiledObjectKey, set[CompiledObjectKey]] = {} - downstream_model_deps: dict[CompiledObjectKey, set[CompiledObjectKey]] = {} - for key in model_keys: - upstream_models: set[CompiledObjectKey] = set() - for dependency in graph.upstream_deps.get(key, ()): - if dependency in model_keys: - upstream_models.add(dependency) - remaining_model_deps[key] = upstream_models - downstream_models: set[CompiledObjectKey] = set() - for dependency in graph.downstream_deps.get(key, ()): - if dependency in model_keys: - downstream_models.add(dependency) - downstream_model_deps[key] = downstream_models - current_layer: set[CompiledObjectKey] = { - key for key, deps in remaining_model_deps.items() if not deps - } - visited: set[CompiledObjectKey] = set() - layer_count: int = 0 - while current_layer: - layer_count += 1 - next_layer: set[CompiledObjectKey] = set() - for key in current_layer: - visited.add(key) - for downstream_key in downstream_model_deps.get(key, set()): - if downstream_key in visited: - continue - remaining_model_deps[downstream_key].discard(key) - if not remaining_model_deps[downstream_key]: - next_layer.add(downstream_key) - current_layer = next_layer - if len(visited) != len(model_keys): - return max(layer_count, 1) - return layer_count + return graph.native.model_layer_count() def _relative_target_path(path: Path) -> str: diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index c30b70555b..a1618c6426 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -2,7 +2,6 @@ from __future__ import annotations -import os import time from dataclasses import replace from functools import partial @@ -23,13 +22,13 @@ write_static_compile_target, ) from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter +from sqlbuild.cli.commands.classes.native_artifact_batch import NativeArtifactBatch from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.models import ( CompileAnalysis, CompileWriteResult, ) -from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES from sqlbuild.cli.entry.models import CompileProfileFlags from sqlbuild.cli.output.models import ( WrittenTarget, @@ -46,13 +45,13 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.models import ProjectColumnLineage +from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.main.selected_graph import ( build_project_graph_with_analysis_selection, ) from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.rule_engine.classes.early_sql_lint import EarlySqlLint from sqlbuild.rule_engine.constants import RULE_FIX_AVAILABLE_NOTE @@ -186,14 +185,8 @@ def _analyze_compile_project( _ = complete_compile_phase( status=status, message=f"Validated model contracts. ({contract_ms / 1000:.2f}s)" ) - selected_keys: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=select, - exclude=exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, + selected_keys: frozenset[CompiledObjectKey] = select_project_graph( + graph=graph, select=select, exclude=exclude ) rules_result: RulesRunResult = RulesRunResult( findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0 @@ -278,19 +271,19 @@ def build_compile_manifest_payload( if not manifest: return None - from sqlbuild.compiler.manifest.main.build import build_manifest + from sqlbuild.compiler.manifest.main.build_graph_manifest import build_graph_manifest manifest_start: float = time.monotonic() _ = start_compile_phase(status=status, message="Building manifest...") - manifest_payload: dict[str, object] = build_manifest( + manifest_payload: dict[str, object] = build_graph_manifest( project=analysis.graph.project, + plan_output=None, project_name=analysis.discovered_inputs.project_config.name, adapter_type=resolve_effective_adapter_name( project_config=analysis.discovered_inputs.project_config, local_config=analysis.discovered_inputs.local_config, ), - upstream_deps=analysis.graph.upstream_deps, - downstream_deps=analysis.graph.downstream_deps, + graph=analysis.graph.native, ) _ = complete_compile_phase( status=status, message=f"Built manifest. ({time.monotonic() - manifest_start:.2f}s)" @@ -323,16 +316,14 @@ def write_compile_dag_artifact( project_dir=project_dir, dag_path=dag_path, ) - resolved_dag_path.parent.mkdir(parents=True, exist_ok=True) dag_json: str = build_dag_json( graph=analysis.graph, project_name=analysis.discovered_inputs.project_config.name, python_graph=python_graph, ) - _ = resolved_dag_path.write_text(dag_json, encoding="utf-8") - COMPILE_ARTIFACT_WRITES.written( - path=resolved_dag_path, contents=dag_json.replace("\n", os.linesep).encode("utf-8") - ) + dag_batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=False) + dag_batch.queue(path=resolved_dag_path, contents=dag_json.encode("utf-8")) + dag_batch.flush() _ = complete_compile_phase( status=status, message=f"Wrote DAG artifact. ({time.monotonic() - dag_start:.2f}s)" ) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py b/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py index 1ce65484c7..1710171b86 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py @@ -1,8 +1,8 @@ """One actionable semantic-check notice for human and machine commands.""" from sqlbuild.compiler.compile.main.semantic_coverage import get_semantic_coverage -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType _DISPLAY_LIMIT: int = 10 _ACTIONS: dict[str, str] = { @@ -20,7 +20,7 @@ def selected_semantic_coverage( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None, sql_validation_enabled: bool = True, ) -> dict[str, tuple[str, ...]]: @@ -38,7 +38,7 @@ def selected_semantic_coverage( def semantic_coverage_notice( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None = None, sql_validation_enabled: bool = True, ) -> str | None: diff --git a/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py b/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py index 64c5ccd69f..5e18142ca8 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py @@ -20,12 +20,11 @@ ) from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, - CompiledModel, CompiledModelSqlTestPayload, - CompiledProject, CompiledSqlTest, CompileSqlTestCte, ) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView _CACHE_VERSION: int = 4 _ALGORITHM_FINGERPRINT: str = "sql-test-artifact-v4-orjson-identity" @@ -37,7 +36,7 @@ def build_sql_test_artifact_identity_context( - *, project: CompiledProject, adapter: BaseAdapter + *, project: CompiledProjectView, adapter: BaseAdapter ) -> SqlTestArtifactIdentityContext: """Hash adapter and model inputs once for all test artifact identities.""" @@ -204,7 +203,7 @@ def build_sql_test_artifact_cache_record( ) -def _model_identity(*, model: CompiledModel) -> str: +def _model_identity(*, model: CompiledModelView) -> str: return _payload_digest( { "name": model.name, diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index f8996b2463..46bc33fbc3 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -2,14 +2,13 @@ from __future__ import annotations -import errno import json import os -import stat -import uuid from collections.abc import Callable +from functools import partial from pathlib import Path +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.compile.sql_test_artifact_cache import ( artifact_matches_cache_record, @@ -20,6 +19,7 @@ sql_test_artifact_record_key, write_sql_test_artifact_cache, ) +from sqlbuild.cli.commands.classes.native_artifact_batch import NativeArtifactBatch from sqlbuild.cli.commands.exceptions import StagedArtifactsChangedError from sqlbuild.cli.compile.models import ( PendingStaticSqlTest, @@ -37,16 +37,18 @@ from sqlbuild.cli.paths.main._sql_test_output_path import sql_test_output_path from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, CompiledSqlTest, CompilerDiagnostic, ) from sqlbuild.compiler.compile.types import ( + CompiledProjectView, CompiledResourceType, DiagnosticPhase, DiagnosticSeverity, FunctionLanguage, ) +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.execution.sql_test_model_chain import ( sql_test_model_chain_names_by_key, @@ -68,8 +70,9 @@ _TESTS_DIR: str = "tests" _MANIFEST_FILE: str = "manifest.json" _SQL_FILE_SUFFIX: str = ".sql" -_POSIX_LINE_SEPARATOR: str = "\n" _SQL_TEST_PLANNING_ERROR_CODE: str = PlannerInputError.code +_PUBLISHED_STAGING_CHANGED: str = "changed" +_PUBLISHED_TREE: str = "tree" def write_compile_target( @@ -83,30 +86,16 @@ def write_compile_target( remove_stale_files: bool = (target_dir / _COMPILED_DIR).is_dir() target_dir.mkdir(parents=True, exist_ok=True) - managed_paths: set[Path] = set().union( - _write_models( - target_dir=target_dir, - plan_output=plan_output, - check_existing=remove_stale_files, - ), - _write_functions( - target_dir=target_dir, - adapter=adapter, - plan_output=plan_output, - check_existing=remove_stale_files, - ), - _write_audits( - target_dir=target_dir, - plan_output=plan_output, - check_existing=remove_stale_files, - ), - _write_tests( - target_dir=target_dir, - adapter=adapter, - plan_output=plan_output, - check_existing=remove_stale_files, - ), - ) + batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=remove_stale_files) + managed_paths: set[Path] = set() + for write_group in ( + partial(_write_models, target_dir=target_dir, plan_output=plan_output), + partial(_write_functions, target_dir=target_dir, adapter=adapter, plan_output=plan_output), + partial(_write_audits, target_dir=target_dir, plan_output=plan_output), + partial(_write_tests, target_dir=target_dir, adapter=adapter, plan_output=plan_output), + ): + managed_paths.update(write_group(batch=batch)) + batch.flush() if remove_stale_files: with record_compile_timing("stale_traversal_ms"): _remove_stale_compiled_files(target_dir=target_dir, managed_paths=managed_paths) @@ -127,7 +116,7 @@ def write_static_compile_target( *, target_dir: Path, adapter: BaseAdapter, - project: CompiledProject, + project: CompiledProjectView, manifest: dict[str, object] | None = None, planned_tests: Callable[[], PlannedStaticSqlTests] | None = None, ) -> WrittenTarget: @@ -135,30 +124,21 @@ def write_static_compile_target( remove_stale_files: bool = (target_dir / _COMPILED_DIR).is_dir() target_dir.mkdir(parents=True, exist_ok=True) - managed_paths: set[Path] = set().union( - _write_static_models( - target_dir=target_dir, - project=project, - check_existing=remove_stale_files, - ), - _write_static_functions( - target_dir=target_dir, - adapter=adapter, - project=project, - check_existing=remove_stale_files, - ), - _write_static_audits( - target_dir=target_dir, - project=project, - check_existing=remove_stale_files, - ), - ) + batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=remove_stale_files) + managed_paths: set[Path] = set() + for write_group in ( + partial(_write_static_models, target_dir=target_dir, project=project), + partial(_write_static_functions, target_dir=target_dir, adapter=adapter, project=project), + partial(_write_static_audits, target_dir=target_dir, project=project), + ): + managed_paths.update(write_group(batch=batch)) + batch.flush() test_paths, test_diagnostics = _write_static_tests( target_dir=target_dir, adapter=adapter, project=project, - check_existing=remove_stale_files, planned_tests=planned_tests, + batch=batch, ) managed_paths.update(test_paths) if remove_stale_files: @@ -181,9 +161,7 @@ def write_static_compile_target( def staged_artifact_files(*, target_dir: Path) -> frozenset[str]: """Return the relative paths of every compiled artifact staged under target_dir.""" - return frozenset( - relative for _, relative in _staged_files(staged_dir=target_dir / _COMPILED_DIR) - ) + return frozenset(_native.staged_artifact_files(target_dir / _COMPILED_DIR)) def publish_static_compile_target( @@ -195,28 +173,11 @@ def publish_static_compile_target( ) -> WrittenTarget: """Publish staged files with the same unchanged-file and stale-file semantics.""" - compiled_dir: Path = target_dir / _COMPILED_DIR - staged_dir: Path = prepared.target_dir / _COMPILED_DIR - staged_files: list[tuple[Path, str]] = _staged_files(staged_dir=staged_dir) - if frozenset(relative for _, relative in staged_files) != expected_files: - raise StagedArtifactsChangedError( - "staged compile artifacts changed before publication; no artifact was published", - help="Rerun the compile; avoid deleting target/ while a compile is running.", - ) - target_dir.mkdir(parents=True, exist_ok=True) - moved: bool = not compiled_dir.is_dir() and _move_staged_tree( - staged_dir=staged_dir, path=compiled_dir + _publish_staged( + staged_dir=prepared.target_dir / _COMPILED_DIR, + target_dir=target_dir, + expected_files=expected_files, ) - if not moved: - check_existing: bool = compiled_dir.is_dir() - managed_paths: set[Path] = set() - for source, relative in staged_files: - path: Path = compiled_dir / relative - _publish_staged_file(source=source, path=path, check_existing=check_existing) - managed_paths.add(path) - if check_existing: - with record_compile_timing("stale_traversal_ms"): - _remove_stale_compiled_files(target_dir=target_dir, managed_paths=managed_paths) if manifest is not None: _write_manifest(target_dir=target_dir, manifest=manifest) return WrittenTarget( @@ -230,89 +191,62 @@ def publish_static_compile_target( ) -def _move_staged_tree(*, staged_dir: Path, path: Path) -> bool: +def _publish_staged(*, staged_dir: Path, target_dir: Path, expected_files: frozenset[str]) -> None: + compiled_dir: Path = target_dir / _COMPILED_DIR + existed: bool = compiled_dir.is_dir() with record_compile_timing("physical_write_ms"): - try: - os.rename(staged_dir, path) - except OSError: - return False - COMPILE_ARTIFACT_WRITES.moved_tree(source=staged_dir, destination=path) - return True - - -def _staged_files(*, staged_dir: Path) -> list[tuple[Path, str]]: - root_prefix_length: int = len(os.fspath(staged_dir)) + 1 - files: list[tuple[Path, str]] = [] - for root, _, filenames in os.walk(staged_dir): - relative_root: str = root[root_prefix_length:] - for filename in filenames: - files.append((Path(root, filename), os.path.join(relative_root, filename))) - return files + kind, published = _native.publish_staged_artifacts( + staged_dir, compiled_dir, sorted(expected_files) + ) + if kind == _PUBLISHED_STAGING_CHANGED: + raise StagedArtifactsChangedError( + "staged compile artifacts changed before publication; no artifact was published", + help="Rerun the compile; avoid deleting target/ while a compile is running.", + ) + if kind == _PUBLISHED_TREE: + COMPILE_ARTIFACT_WRITES.moved_tree(source=staged_dir, destination=compiled_dir) + report_native_answer( + stage=NativeStage.COMPILE_OUTPUTS, kind="published_files", units=len(expected_files) + ) + return + for source, path in published: + COMPILE_ARTIFACT_WRITES.moved(source=source, destination=path) + report_native_answer( + stage=NativeStage.COMPILE_OUTPUTS, kind="published_files", units=len(published) + ) + if existed: + with record_compile_timing("stale_traversal_ms"): + _remove_stale_compiled_files( + target_dir=target_dir, + managed_paths={Path(path) for _, path in published}, + ) -def _publish_staged_file(*, source: Path, path: Path, check_existing: bool) -> None: - with record_compile_timing("physical_write_ms"): - COMPILE_ARTIFACT_WRITES.moved(source=source, destination=path) - if check_existing and path.is_file(): - contents: bytes = source.read_bytes() - existing: bytes = path.read_bytes() - if existing == contents: - return - _ = existing.decode("utf-8") - _overwrite_bytes(path=path, contents=contents) - return - try: - _move_new_file(source=source, path=path) - except FileNotFoundError: - path.parent.mkdir(parents=True, exist_ok=True) - _move_new_file(source=source, path=path) - - -def _move_new_file(*, source: Path, path: Path) -> None: - try: - os.replace(source, path) - except OSError as error: - if error.errno != errno.EXDEV: - raise - _copy_new_file(source=source, path=path) - - -def _copy_new_file(*, source: Path, path: Path) -> None: - """Copy across filesystems through a sibling temporary file so the new file is atomic.""" - - temporary: Path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") - descriptor: int = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666) - try: - try: - _write_all(descriptor=descriptor, path=path, contents=source.read_bytes()) - finally: - os.close(descriptor) - os.replace(temporary, path) - except BaseException: - temporary.unlink(missing_ok=True) - raise - - -def _write_models(*, target_dir: Path, plan_output: PlanOutput, check_existing: bool) -> set[Path]: +def _write_models( + *, target_dir: Path, plan_output: PlanOutput, batch: NativeArtifactBatch +) -> set[Path]: """Write model resolved SQL.""" managed_paths: set[Path] = set() for entry in plan_output.model_entries: compiled_path: Path = target_dir / _COMPILED_DIR / _model_output_path(entry.relative_path) - _write_sql(path=compiled_path, sql=entry.resolved_sql, check_existing=check_existing) + _write_sql(path=compiled_path, sql=entry.resolved_sql, batch=batch) managed_paths.add(compiled_path) return managed_paths def _write_static_models( - *, target_dir: Path, project: CompiledProject, check_existing: bool + *, + target_dir: Path, + project: CompiledProjectView, + batch: NativeArtifactBatch, ) -> set[Path]: """Write offline model query SQL.""" managed_paths: set[Path] = set() for model in project.models: compiled_path: Path = target_dir / _COMPILED_DIR / _model_output_path(model.relative_path) - _write_sql(path=compiled_path, sql=model.query_sql, check_existing=check_existing) + _write_sql(path=compiled_path, sql=model.query_sql, batch=batch) managed_paths.add(compiled_path) return managed_paths @@ -322,7 +256,7 @@ def _write_functions( target_dir: Path, adapter: BaseAdapter, plan_output: PlanOutput, - check_existing: bool, + batch: NativeArtifactBatch, ) -> set[Path]: """Write executable SQL function DDL.""" @@ -349,7 +283,7 @@ def _write_functions( _write_sql( path=function_path, sql=";\n\n".join(statements), - check_existing=check_existing, + batch=batch, ) managed_paths.add(function_path) return managed_paths @@ -359,8 +293,8 @@ def _write_static_functions( *, target_dir: Path, adapter: BaseAdapter, - project: CompiledProject, - check_existing: bool, + project: CompiledProjectView, + batch: NativeArtifactBatch, ) -> set[Path]: """Write offline rendered SQL function DDL.""" @@ -389,13 +323,15 @@ def _write_static_functions( _write_sql( path=function_path, sql=";\n\n".join(statements), - check_existing=check_existing, + batch=batch, ) managed_paths.add(function_path) return managed_paths -def _write_audits(*, target_dir: Path, plan_output: PlanOutput, check_existing: bool) -> set[Path]: +def _write_audits( + *, target_dir: Path, plan_output: PlanOutput, batch: NativeArtifactBatch +) -> set[Path]: """Write resolved audit SQL.""" managed_paths: set[Path] = set() @@ -403,13 +339,16 @@ def _write_audits(*, target_dir: Path, plan_output: PlanOutput, check_existing: folder: Path = _audit_folder(entry) file_name: str = _audit_file_name(entry) audit_path: Path = target_dir / _COMPILED_DIR / _AUDITS_DIR / folder / file_name - _write_sql(path=audit_path, sql=entry.resolved_sql, check_existing=check_existing) + _write_sql(path=audit_path, sql=entry.resolved_sql, batch=batch) managed_paths.add(audit_path) return managed_paths def _write_static_audits( - *, target_dir: Path, project: CompiledProject, check_existing: bool + *, + target_dir: Path, + project: CompiledProjectView, + batch: NativeArtifactBatch, ) -> set[Path]: """Write offline resolved audit SQL.""" @@ -422,7 +361,7 @@ def _write_static_audits( attached_column_name=audit.attached_column_name, ) audit_path: Path = target_dir / _COMPILED_DIR / _AUDITS_DIR / folder / file_name - _write_sql(path=audit_path, sql=audit.sql_body, check_existing=check_existing) + _write_sql(path=audit_path, sql=audit.sql_body, batch=batch) managed_paths.add(audit_path) return managed_paths @@ -432,7 +371,7 @@ def _write_tests( target_dir: Path, adapter: BaseAdapter, plan_output: PlanOutput, - check_existing: bool, + batch: NativeArtifactBatch, ) -> set[Path]: """Write resolved SQL-native test SQL.""" @@ -445,7 +384,7 @@ def _write_tests( set_difference_operator=adapter.render_set_difference_operator(), sql_analysis_dialect=adapter.sql_analysis_dialect(), ) - _write_sql(path=test_path, sql=comparison_sql, check_existing=check_existing) + _write_sql(path=test_path, sql=comparison_sql, batch=batch) managed_paths.add(test_path) return managed_paths @@ -454,7 +393,7 @@ def static_sql_test_planning_diagnostics( *, target_dir: Path, adapter: BaseAdapter, - project: CompiledProject, + project: CompiledProjectView, planned_tests: Callable[[], PlannedStaticSqlTests] | None = None, ) -> tuple[CompilerDiagnostic, ...]: """Plan uncached SQL tests and report their errors without writing artifacts or cache.""" @@ -469,7 +408,7 @@ def static_sql_test_planning_diagnostics( def plan_static_sql_tests( - *, target_dir: Path, adapter: BaseAdapter, project: CompiledProject + *, target_dir: Path, adapter: BaseAdapter, project: CompiledProjectView ) -> PlannedStaticSqlTests: """Plan every SQL test whose cached artifact under target_dir cannot be reused.""" @@ -492,7 +431,7 @@ def _resolve_planned_static_tests( *, target_dir: Path, adapter: BaseAdapter, - project: CompiledProject, + project: CompiledProjectView, planned_tests: Callable[[], PlannedStaticSqlTests] | None, ) -> PlannedStaticSqlTests: planned: PlannedStaticSqlTests | None = planned_tests() if planned_tests is not None else None @@ -502,7 +441,7 @@ def _resolve_planned_static_tests( def _partition_cached_static_tests( - *, tests_root: Path, adapter: BaseAdapter, project: CompiledProject + *, tests_root: Path, adapter: BaseAdapter, project: CompiledProjectView ) -> tuple[ tuple[PendingStaticSqlTest, ...], set[Path], @@ -563,7 +502,7 @@ def _partition_cached_static_tests( def _plan_static_test_artifacts( - *, adapter: BaseAdapter, project: CompiledProject, tests: tuple[CompiledSqlTest, ...] + *, adapter: BaseAdapter, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] ) -> tuple[NativeSqlTestArtifact, ...]: if not tests: return () @@ -584,9 +523,9 @@ def _write_static_tests( *, target_dir: Path, adapter: BaseAdapter, - project: CompiledProject, - check_existing: bool, + project: CompiledProjectView, planned_tests: Callable[[], PlannedStaticSqlTests] | None, + batch: NativeArtifactBatch, ) -> tuple[set[Path], tuple[CompilerDiagnostic, ...]]: """Write offline SQL-native test SQL and report uncached planning errors.""" @@ -597,16 +536,22 @@ def _write_static_tests( tests_root: Path = planned.tests_root managed_paths: set[Path] = set(planned.cached_paths) current_records: dict[str, SqlTestArtifactCacheRecord] = dict(planned.cached_records) + test_paths: list[Path] = [] for pending, artifact in zip(planned.pending, planned.artifacts, strict=True): - test: CompiledSqlTest = pending.test - record_key: str | None = pending.record_key - artifact_identity: str | None = pending.artifact_identity test_path: Path = tests_root / compiled_sql_test_output_path( - test=test, + test=pending.test, model_names=artifact.model_names, ) - _write_sql(path=test_path, sql=artifact.sql, check_existing=check_existing) + _write_sql(path=test_path, sql=artifact.sql, batch=batch) managed_paths.add(test_path) + test_paths.append(test_path) + batch.flush() + for pending, artifact, test_path in zip( + planned.pending, planned.artifacts, test_paths, strict=True + ): + test: CompiledSqlTest = pending.test + record_key: str | None = pending.record_key + artifact_identity: str | None = pending.artifact_identity artifact_diagnostics: tuple[CompilerDiagnostic, ...] = _sql_test_artifact_diagnostics( test=test, artifact=artifact ) @@ -650,120 +595,24 @@ def _write_manifest(*, target_dir: Path, manifest: dict[str, object]) -> None: """Write manifest.json.""" manifest_path: Path = target_dir / _MANIFEST_FILE - _write_text_if_changed(path=manifest_path, contents=json.dumps(manifest, indent=2) + "\n") + batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=True) + batch.queue(path=manifest_path, contents=(json.dumps(manifest, indent=2) + "\n").encode()) + batch.flush() -def _write_sql(*, path: Path, sql: str, check_existing: bool = True) -> None: - """Write one SQL file.""" +def _write_sql(*, path: Path, sql: str, batch: NativeArtifactBatch) -> None: + """Queue one SQL file for the native writer.""" - contents: str = sql.rstrip() + "\n" - if os.linesep != _POSIX_LINE_SEPARATOR: - _write_text_if_changed( - path=path, - contents=contents, - check_existing=check_existing, - ) - return - _write_bytes_if_changed( - path=path, - contents=contents.encode("utf-8"), - check_existing=check_existing, - ) - - -def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = True) -> None: - with record_compile_timing("physical_write_ms"): - COMPILE_ARTIFACT_WRITES.written( - path=path, - contents=contents.replace(_POSIX_LINE_SEPARATOR, os.linesep).encode("utf-8"), - ) - if check_existing and path.is_file() and path.read_text(encoding="utf-8") == contents: - return - try: - _overwrite_text(path=path, contents=contents) - except FileNotFoundError: - path.parent.mkdir(parents=True, exist_ok=True) - _overwrite_text(path=path, contents=contents) - - -def _write_bytes_if_changed(*, path: Path, contents: bytes, check_existing: bool = True) -> None: - with record_compile_timing("physical_write_ms"): - COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents) - if check_existing: - existing: bytes | None = _read_existing_file(path=path) - if existing == contents: - return - if existing is not None: - _ = existing.decode("utf-8") - try: - _overwrite_bytes(path=path, contents=contents) - except FileNotFoundError: - path.parent.mkdir(parents=True, exist_ok=True) - _overwrite_bytes(path=path, contents=contents) - - -def _read_existing_file(*, path: Path) -> bytes | None: - """Return a regular file's bytes in one open, or None when there is no file to compare.""" - - try: - with open(path, "rb") as handle: - if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): - return None - return handle.read() - except (FileNotFoundError, NotADirectoryError, IsADirectoryError): - return None - - -def _overwrite_text(*, path: Path, contents: str) -> None: - _ = path.write_text(contents, encoding="utf-8") - - -def _overwrite_bytes(*, path: Path, contents: bytes) -> None: - descriptor: int = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o666) - try: - _write_all(descriptor=descriptor, path=path, contents=contents) - finally: - os.close(descriptor) - - -def _write_all(*, descriptor: int, path: Path, contents: bytes) -> None: - offset: int = 0 - while offset < len(contents): - written: int = os.write(descriptor, contents[offset:]) - if written == 0: - raise OSError(f"failed to write compiled artifact '{path}'") - offset += written + batch.queue(path=path, contents=(sql.rstrip() + "\n").encode("utf-8")) def _remove_stale_compiled_files(*, target_dir: Path, managed_paths: set[Path]) -> None: - compiled_dir: Path = target_dir / _COMPILED_DIR - if not compiled_dir.is_dir(): - return - managed_names: set[str] = {os.fspath(path) for path in managed_paths} - removed_directories: set[str] = set() - for root, directories, filenames in os.walk(compiled_dir, topdown=False): - kept_file: bool = False - for filename in filenames: - path: str = os.path.join(root, filename) - if path in managed_names: - kept_file = True - else: - os.unlink(path) - if not kept_file and all( - os.path.join(root, name) in removed_directories for name in directories - ): - if root == os.fspath(compiled_dir) or _remove_empty_directory(root): - removed_directories.add(root) - if os.fspath(compiled_dir) in removed_directories: - _remove_empty_directory(compiled_dir) - - -def _remove_empty_directory(directory: str | Path) -> bool: - try: - os.rmdir(directory) - except OSError: - return False - return True + removed: int = _native.remove_stale_artifacts( + target_dir / _COMPILED_DIR, [os.fspath(path) for path in managed_paths] + ) + report_native_answer( + stage=NativeStage.COMPILE_OUTPUTS, kind="stale_files_removed", units=removed + ) def _model_output_path(relative_path: Path) -> Path: diff --git a/src/sqlbuild/cli/commands/_helpers/contract/progress.py b/src/sqlbuild/cli/commands/_helpers/contract/progress.py index 18f5c1773f..f5cc09938a 100644 --- a/src/sqlbuild/cli/commands/_helpers/contract/progress.py +++ b/src/sqlbuild/cli/commands/_helpers/contract/progress.py @@ -11,7 +11,7 @@ from sqlbuild.cli.progress.classes.connection_progress_reporter import ( ConnectionProgressReporter, ) -from sqlbuild.compiler.compile.models import CompiledModel, CompiledSource +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledSourceView from sqlbuild.compiler.contract_adoption.main.compare import compare_contracts from sqlbuild.compiler.contract_adoption.models import ContractAdoptionResult, ContractEvidence from sqlbuild.compiler.pipeline.models import ProjectGraph @@ -22,8 +22,8 @@ def inspect_contracts_with_progress( *, context: AdapterConnectionContext, request: ContractCommandRequest, - selected_models: tuple[CompiledModel, ...], - selected_sources: tuple[CompiledSource, ...], + selected_models: tuple[CompiledModelView, ...], + selected_sources: tuple[CompiledSourceView, ...], progress_stream: TextIO, use_progress_color: bool, ) -> tuple[ContractEvidence, ...]: diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index a338b0337e..8b8ad248b1 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -19,7 +19,7 @@ from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter from sqlbuild.cli.progress.main._execution_header import format_execution_header from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.models import ( PlanOutput, ) @@ -40,7 +40,7 @@ def execute_sqlbuild_build_work( *, context: DbtSqlbuildWorkContext, command: DbtInteropCommand, - project: CompiledProject, + project: CompiledProjectView, project_dir: Path, fail_fast: bool, verbose: bool, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 917ff920f0..e14a04fe24 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -19,6 +19,7 @@ ) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode +from sqlbuild.cli.entry.main._withhold_launch_record import withhold_launch_record from sqlbuild.cli.entry.models import ( CliEntrypointHandlers, CompileCommandRequest, @@ -53,6 +54,8 @@ def _dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers ) effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() selector_inputs: SelectorInputs = read_selector_file_inputs(args.select_file) + if args.select_file: + withhold_launch_record() select: tuple[str, ...] = (*tuple(args.select), *selector_inputs.selectors) compile_exit_code: int | None = _dispatch_compile_or_contract( args=args, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/observability.py b/src/sqlbuild/cli/commands/_helpers/entry/observability.py index 3da7bf8c49..2200ca0cd3 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/observability.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/observability.py @@ -21,6 +21,7 @@ from sqlbuild.cli.commands.main.entrypoint._dispatch_with_output_capture import ( configured_output_capture_scope, ) +from sqlbuild.cli.entry.main._withhold_launch_record import withhold_launch_record from sqlbuild.cli.output.classes.terminal_event_index import ( TerminalEventIndex, terminal_event_index_scope, @@ -114,6 +115,7 @@ def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> Iterato extensions.command_output_sinks ) if event_exporters or command_output_sinks: + withhold_launch_record() exporter_delivery: EventExporterDispatcher = EventExporterDispatcher( shutdown_timeout_seconds=( DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/selection.py b/src/sqlbuild/cli/commands/_helpers/freshness/selection.py index 2e70b93f96..dd81d5f74a 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/selection.py @@ -4,8 +4,8 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.compiler.planner.main.selection.upstream import expand_project_upstream_keys @@ -16,30 +16,12 @@ def resolve_freshness_source_names( selected_keys: frozenset[CompiledObjectKey] if select: - selected_keys = resolve_project_selectors( - select=select, - exclude=(), - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) + selected_keys = select_project_graph(graph=graph, select=select, exclude=()) else: selected_keys = frozenset(graph.all_keys.values()) excluded_keys: frozenset[CompiledObjectKey] = ( - resolve_project_selectors( - select=exclude, - exclude=(), - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) - if exclude - else frozenset() + select_project_graph(graph=graph, select=exclude, exclude=()) if exclude else frozenset() ) source_names: frozenset[str] = _source_names_for_keys(graph=graph, keys=selected_keys) excluded_source_names: frozenset[str] = _source_names_for_keys( diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py index c8d3ab34a2..1868504ac8 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/compilation.py @@ -14,7 +14,7 @@ JanitorInvocation, ) from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -35,7 +35,7 @@ def compile_janitor_project(*, invocation: JanitorInvocation) -> JanitorCompileC ) with OperationLifecycle(operation_kind="project", operation_name="project_compile"): status.on_progress("Compiling project...") - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=invocation.discovered_inputs, adapter=adapter_context.adapter, selected_target=( diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/cache.py b/src/sqlbuild/cli/commands/_helpers/lineage/cache.py index eef67adaca..89a1101941 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/cache.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/cache.py @@ -2,20 +2,20 @@ from __future__ import annotations -import hashlib import json import os -import re import sqlite3 +import sys import tempfile from contextlib import closing from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from typing import Any +import sqlbuild._native as _native from sqlbuild.cli.commands.constants import TARGET_DIRECTORY_NAME from sqlbuild.cli.commands.models import LineageNode, RelationLineageIndex -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.constants import CACHE_DIRECTORY_NAME from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name from sqlbuild.compiler.pipeline.models import ProjectGraph @@ -27,27 +27,10 @@ _CACHE_MAX_BYTES: int = 50_000_000 _CACHE_MAX_ROWS: int = 1_000_000 _SQLITE_TIMEOUT_SECONDS: float = 5.0 -_FINGERPRINT_SUFFIXES: frozenset[str] = frozenset({".csv", ".py", ".sql", ".toml", ".yaml", ".yml"}) -_FINGERPRINT_ROOT_FILES: frozenset[str] = frozenset({".gitignore", ".sqlbuildignore"}) -_EXCLUDED_ROOTS: frozenset[str] = frozenset( - { - ".git", - ".mypy_cache", - ".pytest_cache", - ".ruff_cache", - ".tox", - ".venv", - "node_modules", - "target", - "venv", - } +_FIRST_PYTHON_SKIPPING_INTERRUPTED_LISTINGS: tuple[int, int] = (3, 13) +_INTERRUPTED_LISTING_UNCACHEABLE: bool = ( + sys.version_info < _FIRST_PYTHON_SKIPPING_INTERRUPTED_LISTINGS ) -_EXCLUDED_PATH_PARTS: frozenset[str] = frozenset({"__pycache__"}) -_ENVIRONMENT_PATTERN: re.Pattern[bytes] = re.compile(rb"ENV:\s*([A-Za-z0-9_]+)") -_ENVIRONMENT_MARKER: bytes = b"ENV:" -_DYNAMIC_CONTEXT_MARKER: bytes = b"CTX:" -_DYNAMIC_CONTEXT_GRAPH_SUFFIXES: frozenset[str] = frozenset({".py", ".toml"}) -_NON_ASCII_BYTE_START: int = 128 def relation_lineage_fingerprint( @@ -56,49 +39,25 @@ def relation_lineage_fingerprint( """Hash authored project inputs and invocation context without retaining their values.""" try: - digest: Any = hashlib.sha256() - digest.update(_CACHE_ALGORITHM_VERSION.encode("ascii")) - digest.update(_sqlbuild_version().encode("utf-8")) - digest.update( - json.dumps( - {} if cli_vars is None else cli_vars, - ensure_ascii=True, - separators=(",", ":"), - sort_keys=True, - ).encode("ascii") - ) - environment_names: set[str] = set() - for path in _authored_files(project_dir=project_dir): - relative_path: str = path.relative_to(project_dir).as_posix() - contents: bytes = path.read_bytes() - if ( - _DYNAMIC_CONTEXT_MARKER in contents - and path.suffix.lower() in _DYNAMIC_CONTEXT_GRAPH_SUFFIXES - ): - return None - environment_matches: list[re.Match[bytes]] = list( - _ENVIRONMENT_PATTERN.finditer(contents) - ) - if contents.count(_ENVIRONMENT_MARKER) != len(environment_matches): - return None - if any( - match.end() < len(contents) and contents[match.end()] >= _NON_ASCII_BYTE_START - for match in environment_matches - ): - return None - environment_names.update( - match.group(1).decode("ascii") for match in environment_matches - ) - digest.update(len(relative_path).to_bytes(8, byteorder="big")) - digest.update(relative_path.encode("utf-8")) - digest.update(len(contents).to_bytes(8, byteorder="big")) - digest.update(contents) - for name in sorted(environment_names): - digest.update(name.encode("ascii")) - digest.update(os.environ.get(name, "").encode("utf-8")) - return digest.hexdigest() - except (OSError, TypeError, UnicodeError, ValueError): + prefix: bytes = _fingerprint_prefix(cli_vars=cli_vars) + except (TypeError, UnicodeError, ValueError): return None + return _native.relation_lineage_fingerprint( + project_dir, prefix, _INTERRUPTED_LISTING_UNCACHEABLE + ) + + +def _fingerprint_prefix(*, cli_vars: dict[str, object] | None) -> bytes: + return ( + _CACHE_ALGORITHM_VERSION.encode("ascii") + + _sqlbuild_version().encode("utf-8") + + json.dumps( + {} if cli_vars is None else cli_vars, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + ) def read_relation_lineage_cache( @@ -157,27 +116,6 @@ def write_relation_lineage_cache( return index -def _authored_files(*, project_dir: Path) -> tuple[Path, ...]: - return tuple( - sorted( - path - for path in project_dir.rglob("*") - if path.is_file() - and not _is_excluded(path=path, project_dir=project_dir) - and ( - path.suffix.lower() in _FINGERPRINT_SUFFIXES or path.name in _FINGERPRINT_ROOT_FILES - ) - ) - ) - - -def _is_excluded(*, path: Path, project_dir: Path) -> bool: - parts: tuple[str, ...] = path.relative_to(project_dir).parts - return ( - not parts or parts[0] in _EXCLUDED_ROOTS or bool(_EXCLUDED_PATH_PARTS.intersection(parts)) - ) - - def _cache_path(*, project_dir: Path) -> Path: return ( project_dir @@ -200,7 +138,7 @@ def _relation_index(*, graph: ProjectGraph) -> RelationLineageIndex: ) -def _lineage_node(*, project: CompiledProject, key: CompiledObjectKey) -> LineageNode: +def _lineage_node(*, project: CompiledProjectView, key: CompiledObjectKey) -> LineageNode: for model in project.models: if model.key == key: return LineageNode( diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py index 0c7f9107f8..d780189e70 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/execution.py @@ -26,7 +26,6 @@ select_selector_lineage, select_target_lineage, ) -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.constants import ( COLUMN_TARGET_SEPARATOR, JSON_OUTPUT_FORMAT, @@ -42,16 +41,12 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.lineage.main.build_semantic_uses import build_direct_semantic_uses from sqlbuild.compiler.lineage.models import DirectSemanticColumnUse -from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.constants import UNKNOWN_SELECTOR_ERROR_CODE from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, @@ -111,6 +106,10 @@ def _prepare_graph( ) if cached is not None: return cached, None, parsed_depth + from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter + from sqlbuild.compiler.discovery.main.discover import discover_project_inputs + from sqlbuild.compiler.pipeline.main.graph import build_project_graph + discovered: DiscoveredProjectInputs = discover_project_inputs( project_dir=project_dir, sql_analysis_enabled_override=(None if requires_compiled_graph else False), @@ -160,6 +159,8 @@ def _semantic_uses( ) -> tuple[DirectSemanticColumnUse, ...]: if not request.include_uses: return () + from sqlbuild.compiler.lineage.main.build_semantic_uses import build_direct_semantic_uses + if not isinstance(graph, ProjectGraph) or adapter is None: raise CliUserError( "semantic uses require a compiled project graph", @@ -215,6 +216,8 @@ def _render_lineage( if error.code == UNKNOWN_SELECTOR_ERROR_CODE and isinstance( graph, RelationLineageIndex ): + from sqlbuild.compiler.discovery.main.discover import discover_project_inputs + reject_unit_test_selectors( select=request.select, exclude=request.exclude, @@ -254,14 +257,10 @@ def _semantic_use_model_names( )[0] for target in request.targets ) - keys: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=request.select, - exclude=request.exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, + from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph + + keys: frozenset[CompiledObjectKey] = select_project_graph( + graph=graph, select=request.select, exclude=request.exclude ) return frozenset(key.name for key in keys if key.resource_type == CompiledResourceType.MODEL) diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/selection.py b/src/sqlbuild/cli/commands/_helpers/lineage/selection.py index db8086d4f7..61f8dfe0bb 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/selection.py @@ -26,9 +26,8 @@ ) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.graph.main.path_nodes import path_nodes from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many @@ -517,7 +516,7 @@ def _build_node( ) -> LineageNode: if isinstance(graph, RelationLineageIndex): return graph.nodes.get(key, LineageNode(key=key)) - project: CompiledProject = graph.project + project: CompiledProjectView = graph.project for model in project.models: if model.key == key: return LineageNode( diff --git a/src/sqlbuild/cli/commands/_helpers/lint/selection.py b/src/sqlbuild/cli/commands/_helpers/lint/selection.py index 4998b9d3a5..1fe414da86 100644 --- a/src/sqlbuild/cli/commands/_helpers/lint/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/lint/selection.py @@ -14,10 +14,10 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.pipeline.main.graph import build_project_graph +from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.lint.constants import LINT_DIRECTORY_NAMES, PARENT_PATH_SEGMENT, SQL_FILE_SUFFIX from sqlbuild.lint.main.scan_fixture_typed_null_candidates import ( scan_fixture_typed_null_candidates, @@ -109,14 +109,8 @@ def _resolve_model_paths( if exact_paths is not None: return exact_paths graph: ProjectGraph = build_project_graph(discovered_inputs=discovered, adapter=adapter) - selected_keys: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=select, - exclude=exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, + selected_keys: frozenset[CompiledObjectKey] = select_project_graph( + graph=graph, select=select, exclude=exclude ) selected_names: frozenset[str] = frozenset( key.name for key in selected_keys if key.resource_type == CompiledResourceType.MODEL diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/command.py b/src/sqlbuild/cli/commands/_helpers/refactor/command.py index f9f5c71f48..0038433670 100644 --- a/src/sqlbuild/cli/commands/_helpers/refactor/command.py +++ b/src/sqlbuild/cli/commands/_helpers/refactor/command.py @@ -11,6 +11,12 @@ from sqlbuild.cli.commands._helpers.refactor.compile_facts import compile_for_refactor from sqlbuild.cli.commands._helpers.refactor.layer_move import layer_move_suggestion +from sqlbuild.cli.commands._helpers.refactor.native import ( + commit_native_refactor, + migrate_native_refactor, + plan_native_refactor, + stage_native_refactor, +) from sqlbuild.cli.commands._helpers.refactor.output import ( render_refactor_json, render_refactor_text, @@ -20,11 +26,6 @@ from sqlbuild.cli.commands.models import RefactorCommandRequest, RefactorCompile from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.refactoring.exceptions import RefactorInputError -from sqlbuild.compiler.refactoring.main.commit_refactor_plan import commit_refactor_plan -from sqlbuild.compiler.refactoring.main.plan_column_rename import plan_column_rename -from sqlbuild.compiler.refactoring.main.plan_model_refactor import plan_model_refactor -from sqlbuild.compiler.refactoring.main.stage_refactor_plan import stage_refactor_plan -from sqlbuild.compiler.refactoring.main.with_model_migration import with_model_migration from sqlbuild.compiler.refactoring.models import RefactorPlan, RefactorRequest from sqlbuild.compiler.refactoring.types import RefactorOperation, RefactorStatus from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter @@ -69,11 +70,9 @@ def _run( request=request, all_keys=before.project.graph.all_keys ) status.start("Planning edits...") - plan: RefactorPlan = ( - plan_column_rename(project=before.project, request=refactor_request) - if refactor_request.operation == RefactorOperation.RENAME_COLUMN - else plan_model_refactor(project=before.project, request=refactor_request) - ) + plan: RefactorPlan + plan_json: str + plan, plan_json = plan_native_refactor(project=before.project, request=refactor_request) status.complete(message=f"Planned edits to {_files(plan)}.") if plan.blocking or plan.manual: status.error("Refused: some references cannot be rewritten safely.") @@ -87,15 +86,17 @@ def _run( with tempfile.TemporaryDirectory(prefix=_STAGING_PREFIX) as staging: staging_dir: Path = Path(staging) status.start("Verifying the edited project compiles...") - originals: dict[str, str] = stage_refactor_plan( - project_dir=project_dir, staging_dir=staging_dir, plan=plan + originals: dict[str, str] = stage_native_refactor( + project_dir=project_dir, staging_dir=staging_dir, plan_json=plan_json, copy_inputs=True ) after: RefactorCompile = compile_for_refactor(project_dir=staging_dir, no_cache=True) if refactor_request.operation != RefactorOperation.RENAME_COLUMN: - migrated: RefactorPlan = with_model_migration( - plan=plan, - before=before.project.graph.project, - after=after.project.graph.project if after.project is not None else None, + migrated: RefactorPlan + migrated_json: str + migrated, migrated_json = migrate_native_refactor( + plan_json=plan_json, + before=before.project, + after=after.project, originals=originals, ) if migrated.blocking: @@ -108,9 +109,12 @@ def _run( use_color=use_color, ) if migrated.migrations != plan.migrations: - plan = migrated - originals = stage_refactor_plan( - project_dir=project_dir, staging_dir=staging_dir, plan=plan, copy_inputs=False + plan, plan_json = migrated, migrated_json + originals = stage_native_refactor( + project_dir=project_dir, + staging_dir=staging_dir, + plan_json=plan_json, + copy_inputs=False, ) after = compile_for_refactor(project_dir=staging_dir, no_cache=True) diagnostics: tuple[CompilerDiagnostic, ...] = _relative( @@ -135,7 +139,7 @@ def _run( use_color=use_color, ) status.start(f"Writing {_files(plan)}...") - commit_refactor_plan(project_dir=project_dir, originals=originals, plan=plan) + commit_native_refactor(project_dir=project_dir, originals=originals, plan_json=plan_json) status.complete(message=f"Wrote {_files(plan)}.") return _finish( request=request, diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/facts.py b/src/sqlbuild/cli/commands/_helpers/refactor/facts.py new file mode 100644 index 0000000000..e0691dfee9 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/facts.py @@ -0,0 +1,291 @@ +"""Plain-data compiler facts for native refactoring; the only `CompiledProject` reader.""" + +from __future__ import annotations + +import json +import sys +import unicodedata +from collections.abc import Callable +from pathlib import Path + +from sqlbuild.compiler.compile.models import ( + CompiledSqlExpansion, + ExpansionSpan, +) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView +from sqlbuild.compiler.discovery.models import ( + DiscoveredAuditBlock, + DiscoveredProjectInputs, + DiscoveredSqlTestBlock, +) +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.refactoring.constants import ( + GENERIC_DIALECT, + MIGRATE_FROM_KEY, + MODEL_KIND_PREFIX, +) +from sqlbuild.compiler.refactoring.main.find_python_string_locations import ( + find_python_string_locations, +) +from sqlbuild.compiler.refactoring.models import ManualLocation, RefactorProject, RefactorRequest +from sqlbuild.compiler.refactoring.types import RefactorOperation, SqlFileRole +from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup +from sqlbuild.compiler.scopes.main.preview_scope_move import preview_scope_move +from sqlbuild.compiler.scopes.main.relocate_declarations_for_move import ( + relocate_declarations_for_move, +) +from sqlbuild.compiler.scopes.models import ( + DeclarationRecord, + MovePreview, + ResourceIdentity, + ScopeDiagnostic, + ScopeIndex, +) +from sqlbuild.compiler.scopes.types import ResourceKind +from sqlbuild.spec.contracts.main.get_config_str import get_config_str +from sqlbuild.spec.contracts.models import SchemaColumn, SourceColumnEntry + +_MATERIALIZED_KEY: str = "materialized" + + +def refactor_facts_json(*, project: RefactorProject, request: RefactorRequest) -> str: + """Return the facts one refactoring plan reads, as the native planner's JSON.""" + + compiled: CompiledProjectView = project.graph.project + discovered: DiscoveredProjectInputs = project.discovered + return json.dumps( + { + "project_dir": str(project.project_dir), + "dialect": compiled.sql_analysis_dialect or GENERIC_DIALECT, + "python_version": list(sys.version_info[:2]), + "unicode_version": unicodedata.unidata_version, + "models": model_facts(project=compiled, project_dir=project.project_dir), + "sources": [ + _columns(name=source.name, columns=source.source_entry.columns) + for source in compiled.sources + ], + "seeds": [ + _columns(name=seed.name, columns=seed.schema_entry.columns) + for seed in compiled.seeds + ], + "model_files": [ + {"path": item.relative_path.as_posix(), "contents": item.contents} + for item in discovered.model_files + ], + "authored_files": _authored_files(discovered=discovered), + "yaml_files": [ + {"path": item.relative_path.as_posix(), "contents": item.contents} + for item in (*discovered.source_files, *discovered.schema_files) + ], + "python_locations": [ + _location(item) for item in _python_locations(project=project, request=request) + ], + } + ) + + +def model_facts_json(*, project: RefactorProject | None) -> str | None: + """Return the model facts of one compiled project as JSON, or None without a project.""" + + if project is None: + return None + return json.dumps(model_facts(project=project.graph.project, project_dir=project.project_dir)) + + +def model_facts(*, project: CompiledProjectView, project_dir: Path) -> list[dict[str, object]]: + """Return every compiled model's refactoring facts.""" + + expansions: dict[Path, CompiledSqlExpansion] = { + path.resolve(): expansion for path, expansion in reversed(project.sql_expansions.items()) + } + return [ + _model(model=model, expansion=expansions.get((project_dir / model.relative_path).resolve())) + for model in project.models + ] + + +def declaration_placement_host(*, project: RefactorProject) -> Callable[[tuple[str, str]], str]: + """Return the callback asking the shared scope placement where moved declarations go.""" + + def placement(move: tuple[str, str]) -> str: + model_name, destination = move + report_native_answer(stage=NativeStage.REFACTORING, kind="declaration_moves") + index: ScopeIndex = project.graph.project.scope_index + preview: MovePreview | None + diagnostics: tuple[ScopeDiagnostic, ...] + preview, diagnostics = preview_scope_move( + lookup=build_scope_lookup(index=index), + resource=f"{MODEL_KIND_PREFIX}{model_name}", + destination=destination, + ) + relocated: tuple[DeclarationRecord, ...] | None = ( + None + if preview is None + else relocate_declarations_for_move( + index=index, + resource=ResourceIdentity(kind=ResourceKind.MODEL, name=model_name), + destination=destination, + ) + ) + return json.dumps( + { + "relocated": None + if preview is None or relocated is None + else [ + {"key": repr(record.identity), "path": record.path} + for record in sorted(relocated, key=lambda item: item.identity) + ], + "diagnostics": [item.message for item in diagnostics], + "declarations": [_declaration(record) for record in index.declarations], + } + ) + + return placement + + +def _declaration(record: DeclarationRecord) -> dict[str, object]: + return { + "key": repr(record.identity), + "label": f"{record.identity.kind.value}:{record.identity.name}", + "path": record.path, + "line": record.line, + "column": record.column, + } + + +def _model( + *, model: CompiledModelView, expansion: CompiledSqlExpansion | None +) -> dict[str, object]: + values: dict[str, object] = dict(model.config.values) + return { + "name": model.name, + "path": model.relative_path.as_posix(), + "deps": [{"resource_type": str(dep.resource_type), "name": dep.name} for dep in model.deps], + "query_sql": model.query_sql, + "authored_query_sql": model.authored_query_sql, + "materialized": get_config_str(values=values, key=_MATERIALIZED_KEY), + "declares_migrate_from": MIGRATE_FROM_KEY in values, + "migrate_from_set": values.get(MIGRATE_FROM_KEY) is not None, + "destination": { + "database": model.destination.database, + "schema": model.destination.schema, + "name": model.destination.name, + }, + "inferred_columns": [column.name for column in model.inferred_columns or ()], + "schema_columns": ( + [ + { + "name": column.name, + "migrate_from": column.migrate_from is not None, + "location": ( + { + "path": column.location.path.as_posix(), + "line": column.location.line, + "column": column.location.column, + } + if column.location is not None + else None + ), + } + for column in model.schema_entry.columns + ] + if model.schema_entry is not None + else None + ), + "expansion": ( + { + "expanded_sql": expansion.expanded_sql, + "passes": [_spans(spans=spans) for spans in expansion.passes], + } + if expansion is not None + else None + ), + } + + +def _spans(*, spans: tuple[ExpansionSpan, ...]) -> list[dict[str, int]]: + return [ + { + "source_start": span.source_start, + "source_end": span.source_end, + "output_start": span.output_start, + "output_end": span.output_end, + } + for span in spans + ] + + +def _authored_files(*, discovered: DiscoveredProjectInputs) -> list[dict[str, object]]: + files: list[tuple[SqlFileRole, Path, str, tuple[str, ...]]] = [ + *( + (SqlFileRole.TEST, item.relative_path, item.contents, _block_bodies(item.blocks)) + for item in discovered.test_files + ), + *( + (SqlFileRole.SCENARIO, item.relative_path, item.contents, (item.sql_body,)) + for item in discovered.scenario_files + ), + *( + (SqlFileRole.AUDIT, item.relative_path, item.contents, _block_bodies(item.blocks)) + for item in discovered.audit_files + ), + *( + (SqlFileRole.HOOK, item.relative_path, item.contents, (item.sql_body,)) + for item in discovered.sql_hook_files + ), + *( + (SqlFileRole.FUNCTION, item.relative_path, item.contents, (item.body_sql,)) + for item in discovered.sql_function_files + ), + *( + (SqlFileRole.SCHEMA, item.relative_path, item.contents, ()) + for item in discovered.model_schema_files + ), + ] + return [ + {"role": role.value, "path": path.as_posix(), "contents": contents, "texts": list(texts)} + for role, path, contents, texts in files + ] + + +def _block_bodies( + blocks: tuple[DiscoveredSqlTestBlock | DiscoveredAuditBlock, ...], +) -> tuple[str, ...]: + return tuple(block.sql_body for block in blocks) + + +def _columns( + *, name: str, columns: tuple[SchemaColumn | SourceColumnEntry, ...] +) -> dict[str, object]: + return {"name": name, "columns": [column.name for column in columns]} + + +def _python_locations( + *, project: RefactorProject, request: RefactorRequest +) -> tuple[ManualLocation, ...]: + old: str = request.model_name + if request.operation == RefactorOperation.RENAME_COLUMN: + column: str = request.column_name or "" + return find_python_string_locations( + project_dir=project.project_dir, + discovered=project.discovered, + names=(column,), + reason=f"Python SQL may read column {column} of {old}; check it by hand", + context=old, + ) + return find_python_string_locations( + project_dir=project.project_dir, + discovered=project.discovered, + names=(old,), + reason=f"Python code names model {old}; update it by hand", + ) + + +def _location(location: ManualLocation) -> dict[str, object]: + return { + "path": location.path, + "line": location.line, + "column": location.column, + "reason": location.reason, + } diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/native.py b/src/sqlbuild/cli/commands/_helpers/refactor/native.py new file mode 100644 index 0000000000..33c863e24c --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/refactor/native.py @@ -0,0 +1,207 @@ +"""Plan, stage, migrate and commit one refactoring natively.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, cast + +import sqlbuild._native as _native +from sqlbuild.cli.commands._helpers.refactor.facts import ( + declaration_placement_host, + model_facts_json, + refactor_facts_json, +) +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.refactoring.exceptions import ( + RefactorEditError, + RefactorInputError, + RefactorIOError, + RefactorValueError, + RefactorWriteError, +) +from sqlbuild.compiler.refactoring.models import ( + FileChange, + ManualLocation, + MigrationDeclaration, + RefactorPlan, + RefactorProject, + RefactorRequest, + TextEdit, +) +from sqlbuild.compiler.refactoring.types import ( + EditKind, + NativeRefactorErrorKind, + RefactorOperation, +) + +_PLANNED_EDITS_KIND: str = "planned_edits" +_MIGRATION_EDITS_KIND: str = "migration_edits" +_PLAN_KEY: str = "plan" +_ERROR_KEY: str = "error" + + +def plan_native_refactor( + *, project: RefactorProject, request: RefactorRequest +) -> tuple[RefactorPlan, str]: + """Plan natively, count its edits, and return the plan with its native JSON.""" + + response: dict[str, Any] = _response( + cast(Any, _native).plan_refactor_json( + refactor_facts_json(project=project, request=request), + json.dumps(_request_payload(request)), + declaration_placement_host(project=project), + ) + ) + payload: dict[str, Any] = _payload(response=response, key=_PLAN_KEY) + plan: RefactorPlan = decode_plan(payload) + report_native_answer( + stage=NativeStage.REFACTORING, + kind=_PLANNED_EDITS_KIND, + units=sum(len(change.edits) for change in plan.changes), + ) + return plan, json.dumps(payload) + + +def stage_native_refactor( + *, project_dir: Path, staging_dir: Path, plan_json: str, copy_inputs: bool +) -> dict[str, str]: + """Stage a native plan in the scratch copy and return the original texts.""" + + response: dict[str, Any] = _response( + cast(Any, _native).stage_refactor_plan_json( + str(project_dir), str(staging_dir), plan_json, copy_inputs + ) + ) + return {path: text for path, text in _payload(response=response, key="originals")} + + +def migrate_native_refactor( + *, + plan_json: str, + before: RefactorProject, + after: RefactorProject | None, + originals: dict[str, str], +) -> tuple[RefactorPlan, str]: + """Return the native plan with model `migrate_from` added whenever its relation moves.""" + + before_json: str | None = model_facts_json(project=before) + response: dict[str, Any] = _response( + cast(Any, _native).with_model_migration_json( + plan_json, + before_json, + model_facts_json(project=after), + json.dumps(list(originals.items())), + ) + ) + payload: dict[str, Any] = _payload(response=response, key=_PLAN_KEY) + migrated: RefactorPlan = decode_plan(payload) + added: int = sum(len(change.edits) for change in migrated.changes) - sum( + len(change.edits) for change in decode_plan(json.loads(plan_json)).changes + ) + report_native_answer(stage=NativeStage.REFACTORING, kind=_MIGRATION_EDITS_KIND, units=added) + return migrated, json.dumps(payload) + + +def commit_native_refactor(*, project_dir: Path, originals: dict[str, str], plan_json: str) -> None: + """Write a verified native plan into the project, restoring every file if a write fails.""" + + _ = _payload( + response=_response( + cast(Any, _native).commit_refactor_plan_json( + str(project_dir), json.dumps(list(originals.items())), plan_json + ) + ), + key="written", + ) + + +def decode_plan(payload: dict[str, Any]) -> RefactorPlan: + """Return the plan dataclasses of one native plan payload.""" + + request: dict[str, Any] = payload["request"] + return RefactorPlan( + request=RefactorRequest( + operation=RefactorOperation(request["operation"]), + model_name=request["model_name"], + new_name=request["new_name"], + column_name=request.get("column_name"), + destination=request.get("destination"), + cascade=bool(request.get("cascade")), + ), + changes=tuple(_change(item) for item in payload["changes"]), + manual=tuple(_manual(item) for item in payload["manual"]), + blocking=tuple(_manual(item) for item in payload["blocking"]), + migrations=tuple( + MigrationDeclaration( + model_name=item["model_name"], + declaration=item["declaration"], + reason=item["reason"], + ) + for item in payload["migrations"] + ), + renamed_columns=tuple( + (str(model), str(old), str(new)) for model, old, new in payload["renamed_columns"] + ), + help=payload.get("help"), + ) + + +def _change(item: dict[str, Any]) -> FileChange: + return FileChange( + path=item["path"], + original_path=item["original_path"], + edits=tuple(_edit(edit) for edit in item["edits"]), + ) + + +def _edit(item: dict[str, Any]) -> TextEdit: + return TextEdit( + start=int(item["start"]), + end=int(item["end"]), + replacement=item["replacement"], + kind=EditKind(item["kind"]), + line=int(item["line"]), + column=int(item["column"]), + before=item["before"], + after=item["after"], + ) + + +def _manual(item: dict[str, Any]) -> ManualLocation: + return ManualLocation( + path=item["path"], line=item.get("line"), column=item.get("column"), reason=item["reason"] + ) + + +def _request_payload(request: RefactorRequest) -> dict[str, object]: + return { + "operation": request.operation.value, + "model_name": request.model_name, + "new_name": request.new_name, + "column_name": request.column_name, + "destination": request.destination, + "cascade": request.cascade, + } + + +def _response(text: str) -> dict[str, Any]: + return cast(dict[str, Any], json.loads(text)) + + +def _payload(*, response: dict[str, Any], key: str) -> Any: + error: dict[str, Any] | None = response.get(_ERROR_KEY) + if error is None: + return response[key] + kind: NativeRefactorErrorKind = NativeRefactorErrorKind(str(error["kind"])) + message: str = str(error["message"]) + if kind is NativeRefactorErrorKind.VALUE: + raise RefactorValueError(message) + if kind is NativeRefactorErrorKind.IO: + raise RefactorIOError(message) + if kind is NativeRefactorErrorKind.EDIT: + raise RefactorEditError(message) + if kind is NativeRefactorErrorKind.WRITE: + raise RefactorWriteError(message, help=error.get("help")) + raise RefactorInputError(message, code=str(error["code"]), help=error.get("help")) diff --git a/src/sqlbuild/cli/commands/_helpers/rules/command.py b/src/sqlbuild/cli/commands/_helpers/rules/command.py index a199df7e45..8ae03adbdc 100644 --- a/src/sqlbuild/cli/commands/_helpers/rules/command.py +++ b/src/sqlbuild/cli/commands/_helpers/rules/command.py @@ -19,9 +19,9 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.graph import build_project_graph +from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.rule_engine.main.build_catalogue import build_catalogue from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.main.render_result import format_result @@ -139,14 +139,8 @@ def _run_rule_selection( project_dir=project_dir, ) graph: ProjectGraph = build_project_graph(discovered_inputs=discovered, adapter=adapter) - selected_keys: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=request.select, - exclude=request.exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, + selected_keys: frozenset[CompiledObjectKey] = select_project_graph( + graph=graph, select=request.select, exclude=request.exclude ) result: RulesRunResult = run_rules( graph=graph, diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/selection.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/selection.py index 9fea233777..2874d350c8 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/selection.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/selection.py @@ -14,14 +14,14 @@ ) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.compile.models import ( - CompiledProject, CompiledSqlScenario, ) +from sqlbuild.compiler.compile.types import CompiledProjectView def select_scenarios( *, - project: CompiledProject, + project: CompiledProjectView, selectors: tuple[str, ...], exclude: tuple[str, ...] = (), project_dir: Path, @@ -71,7 +71,7 @@ def select_scenarios( def _select_scenarios_for_selector( - *, project: CompiledProject, selector: str, project_dir: Path + *, project: CompiledProjectView, selector: str, project_dir: Path ) -> tuple[CompiledSqlScenario, ...]: _validate_scenario_selector(selector) selector_path: Path = Path(selector) diff --git a/src/sqlbuild/cli/commands/_helpers/seed/planning.py b/src/sqlbuild/cli/commands/_helpers/seed/planning.py index 4c4665eef2..38f0fd6dfa 100644 --- a/src/sqlbuild/cli/commands/_helpers/seed/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/seed/planning.py @@ -13,7 +13,7 @@ SeedInvocation, ) from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.compiler.compile.models import CompiledSeed +from sqlbuild.compiler.compile.types import CompiledSeedView from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context from sqlbuild.compiler.pipeline.main.static_result import build_static_pipeline_result @@ -56,7 +56,7 @@ def prepare_seed_execution( adapter=invocation.adapter, options=options, ) - selected_seeds: tuple[CompiledSeed, ...] = tuple( + selected_seeds: tuple[CompiledSeedView, ...] = tuple( seed for seed in context.project.seeds if seed.key in context.scope.selected_keys ) hooks: ConnectionHooks = ConnectionHooks( diff --git a/src/sqlbuild/cli/commands/classes/native_artifact_batch.py b/src/sqlbuild/cli/commands/classes/native_artifact_batch.py new file mode 100644 index 0000000000..2e1fdb05d3 --- /dev/null +++ b/src/sqlbuild/cli/commands/classes/native_artifact_batch.py @@ -0,0 +1,42 @@ +"""Queue compiled artifacts and write each group in one native call.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.profiling.main.record import record_compile_timing + +_POSIX_LINE_SEPARATOR: str = "\n" + + +class NativeArtifactBatch: + """Compiled artifacts queued for the native writer, keeping unchanged files when asked.""" + + def __init__(self, *, check_existing: bool) -> None: + self.check_existing: bool = check_existing + self._files: list[tuple[Path, bytes]] = [] + + def queue(self, *, path: Path, contents: bytes) -> None: + """Queue one artifact with platform line endings, noting its bytes for compile reuse.""" + + if os.linesep != _POSIX_LINE_SEPARATOR: + contents = contents.replace(b"\n", os.linesep.encode()) + COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents) + self._files.append((path, contents)) + + def flush(self) -> None: + """Write every queued artifact natively and empty the queue.""" + + if not self._files: + return + with record_compile_timing("physical_write_ms"): + written, unchanged = _native.write_compiled_artifacts(self._files, self.check_existing) + self._files = [] + report_native_answer( + stage=NativeStage.COMPILE_OUTPUTS, kind="artifact_files", units=written + unchanged + ) diff --git a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py b/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py index 9bbab04ad2..83d9d6b5f3 100644 --- a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py +++ b/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py @@ -23,7 +23,8 @@ from sqlbuild.cli.commands.exceptions import StagedArtifactsChangedError from sqlbuild.cli.compile.models import PlannedStaticSqlTests from sqlbuild.cli.output.models import WrittenTarget -from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView _MIN_PREPARED_ARTIFACT_MODELS: int = 128 _MAX_PREPARED_ARTIFACT_MODELS: int = 5000 @@ -45,7 +46,7 @@ def __init__(self, *, enabled: bool) -> None: self._executor: ThreadPoolExecutor | None = None self._future: Future[_PreparedResult] | None = None self._planned_tests: Future[PlannedStaticSqlTests] | None = None - self._planning_inputs: tuple[CompiledProject, BaseAdapter, Path] | None = None + self._planning_inputs: tuple[CompiledProjectView, BaseAdapter, Path] | None = None def __enter__(self) -> PreparedCompileArtifacts: return self @@ -81,7 +82,7 @@ def remove_abandoned_staging(self, *, target_dir: Path) -> None: def start( self, *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, target_dir: Path, stage_artifacts: bool, @@ -100,7 +101,7 @@ def start( self._start_test_planning(project=project, adapter=adapter, target_dir=target_dir) def _start_test_planning( - self, *, project: CompiledProject, adapter: BaseAdapter, target_dir: Path + self, *, project: CompiledProjectView, adapter: BaseAdapter, target_dir: Path ) -> None: """Plan SQL tests against the published target while the compile continues.""" @@ -116,7 +117,7 @@ def plan() -> PlannedStaticSqlTests: self._planned_tests = self._executor.submit(plan) def _start_staging( - self, *, project: CompiledProject, adapter: BaseAdapter, target_dir: Path + self, *, project: CompiledProjectView, adapter: BaseAdapter, target_dir: Path ) -> bool: """Stage artifacts beside target_dir so publication can move them instead of copying.""" @@ -196,7 +197,7 @@ def planning_diagnostics(self) -> tuple[CompilerDiagnostic, ...] | None: return None -def _stages_artifacts(project: CompiledProject) -> bool: +def _stages_artifacts(project: CompiledProjectView) -> bool: """Staging pays off for uncached, mid-sized projects whose memory stays within budget.""" return ( diff --git a/src/sqlbuild/cli/commands/main/dbt/dbt_sqlbuild_work.py b/src/sqlbuild/cli/commands/main/dbt/dbt_sqlbuild_work.py index 3bc9360d21..79a4263063 100644 --- a/src/sqlbuild/cli/commands/main/dbt/dbt_sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/main/dbt/dbt_sqlbuild_work.py @@ -6,7 +6,7 @@ from sqlbuild.cli.commands._helpers.dbt.sqlbuild_work import execute_sqlbuild_build_work from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt.types import DbtInteropCommand @@ -14,7 +14,7 @@ def execute_dbt_sqlbuild_work( *, context: DbtSqlbuildWorkContext, command: DbtInteropCommand, - project: CompiledProject, + project: CompiledProjectView, project_dir: Path, fail_fast: bool, verbose: bool, diff --git a/src/sqlbuild/cli/commands/main/inspection/_contract.py b/src/sqlbuild/cli/commands/main/inspection/_contract.py index b321563017..5f90a9a9a9 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_contract.py +++ b/src/sqlbuild/cli/commands/main/inspection/_contract.py @@ -18,10 +18,12 @@ from sqlbuild.cli.commands.models import AdapterConnectionContext, ContractCommandRequest from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, - CompiledModel, CompiledObjectKey, - CompiledProject, - CompiledSource, +) +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledSourceView, ) from sqlbuild.compiler.contract_adoption.models import ContractAdoptionResult, ContractEvidence from sqlbuild.compiler.contract_adoption.types import ContractAction @@ -29,11 +31,9 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.pipeline.main.project_graph import build_project_graph_from_compiled_project +from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection.selection import ( - resolve_project_selectors, -) from sqlbuild.presentation.main.supports_color import supports_color @@ -58,7 +58,7 @@ def run_contract(request: ContractCommandRequest) -> int: selected_target=None, cli_vars=request.cli_vars, ) - project: CompiledProject = build_compiled_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered, adapter=context.adapter, selected_target=request.from_target, @@ -67,19 +67,13 @@ def run_contract(request: ContractCommandRequest) -> int: analysis_selection=CompileAnalysisSelection(no_cache=True), ) graph: ProjectGraph = build_project_graph_from_compiled_project(project=project) - selected: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=request.select, - exclude=request.exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, + selected: frozenset[CompiledObjectKey] = select_project_graph( + graph=graph, select=request.select, exclude=request.exclude ) - selected_models: tuple[CompiledModel, ...] = tuple( + selected_models: tuple[CompiledModelView, ...] = tuple( model for model in graph.project.models if model.key in selected ) - selected_sources: tuple[CompiledSource, ...] = tuple( + selected_sources: tuple[CompiledSourceView, ...] = tuple( source for source in graph.project.sources if source.key in selected ) if not selected_models and not selected_sources: diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index da69ddaa54..ebb8bf7f05 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -6,13 +6,8 @@ from dataclasses import dataclass, field from datetime import datetime from pathlib import Path -from typing import Any, TextIO +from typing import TYPE_CHECKING, Any, TextIO -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks -from sqlbuild.cli.commands.classes.direct_python_lifecycle_state import ( - DirectPythonLifecycleState, -) from sqlbuild.cli.commands.types import ( DebugCheckStatus, FreshnessSourceStatus, @@ -39,50 +34,10 @@ SkillUpdateResult, WrittenTarget, ) -from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter -from sqlbuild.cli.progress.classes.connection_progress_reporter import ( - ConnectionProgressReporter, -) -from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( - NestedCommandProgressCallbacks, -) -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.cli.progress.models import AuditDisplayEntry, ExecutionCounts # noqa: F401 -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, - CompiledSqlScenario, - CompilerDiagnostic, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredCheckFunction, - DiscoveredProjectInputs, -) -from sqlbuild.compiler.lineage.models import ( - ColumnLineageEdge, - QualifiedLineageColumn, -) from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline.models import ( - ClonePipelineResult, - CompilePipelineResult, - PythonPlanEntry, -) -from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan -from sqlbuild.compiler.refactoring.models import RefactorProject from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus from sqlbuild.cost.types import CostStatus -from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult -from sqlbuild.executor.clone.models import CloneExecutionResult -from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits -from sqlbuild.executor.janitor.models import JanitorPlan -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult -from sqlbuild.integrations.dbt.models import DbtInitRequest -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.runtime.contracts.types import NodeStartCallback from sqlbuild.spec.contracts.models import ( CostConfig, ExecutionLimitsConfig, @@ -90,6 +45,47 @@ SourceEntry, ) +if TYPE_CHECKING: + from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter + from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks + from sqlbuild.cli.commands.classes.direct_python_lifecycle_state import ( + DirectPythonLifecycleState, + ) + from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter + from sqlbuild.cli.progress.classes.connection_progress_reporter import ( + ConnectionProgressReporter, + ) + from sqlbuild.cli.progress.classes.nested_command_progress_callbacks import ( + NestedCommandProgressCallbacks, + ) + from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter + from sqlbuild.compiler.compile.models import ( + CompiledObjectKey, + CompiledSqlScenario, + CompilerDiagnostic, + ) + from sqlbuild.compiler.compile.types import CompiledProjectView + from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs + from sqlbuild.compiler.lineage.models import ColumnLineageEdge, QualifiedLineageColumn + from sqlbuild.compiler.pipeline.models import ( + ClonePipelineResult, + CompilePipelineResult, + PythonPlanEntry, + ) + from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput + from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan + from sqlbuild.compiler.refactoring.models import RefactorProject + from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult + from sqlbuild.executor.clone.models import CloneExecutionResult + from sqlbuild.executor.diff.models import DiffExecutionResult, FullDiffSizeLimits + from sqlbuild.executor.janitor.models import JanitorPlan + from sqlbuild.executor.load.models import LoadExecutionResult + from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult + from sqlbuild.integrations.dbt.models import DbtInitRequest + from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter + from sqlbuild.python_nodes.models import SqlResourceRef + from sqlbuild.runtime.contracts.types import NodeStartCallback + @dataclass(frozen=True) class AuditCommandRequest: @@ -138,7 +134,7 @@ class BuildRunContext: """Resolved inputs needed to render one build execution context.""" command: str - project: CompiledProject + project: CompiledProjectView plan: PlanOutput discovered_inputs: DiscoveredProjectInputs python_plan_entries: tuple[PythonPlanEntry, ...] @@ -707,7 +703,7 @@ class JanitorCompileContext: adapter_name: str adapter: BaseAdapter - project: CompiledProject + project: CompiledProjectView connection_config: dict[str, object] diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py index 5b9872a59f..90bc5d2537 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -3,65 +3,12 @@ from __future__ import annotations import os -import sys import time -from dataclasses import replace -from pathlib import Path -from sqlbuild.cli.compile_reuse._helpers.entry_file import ( - read_entry_header, - read_entry_stdout, - rewrite_entry_inputs, -) -from sqlbuild.cli.compile_reuse._helpers.project_files import ( - carried_forward_digests, - compare_project_files, - needs_refresh, - restamped_paths, - snapshot_project_files, - stored_project_files, - with_missing_digests, -) -from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( - entry_slot_name, - environment_digest, - invocation_digest, - module_stamps_unchanged, - runtime_identity, - search_path_stamps, - settings_inputs_digest, - tracked_environment_names, -) -from sqlbuild.cli.compile_reuse._helpers.target_files import target_files_unchanged -from sqlbuild.cli.compile_reuse._helpers.timings import ( - elapsed_ms, - replace_compile_timings, - reuse_timings, -) -from sqlbuild.cli.compile_reuse.constants import ( - MILLISECONDS_PER_SECOND, - PROJECT_CONFIG_FILENAMES, - REUSE_DISABLE_ENV_VAR, - REUSE_DISABLE_VALUE, - REUSE_ENTRY_DIRECTORY_NAME, - REUSE_ENTRY_SUFFIX, - REUSE_HIT_MESSAGE, - TOTAL_TIMING, -) -from sqlbuild.cli.compile_reuse.models import ( - CompileReuseAttempt, - CompileReuseRequest, - ProjectFilesComparison, - StoredCompileHeader, - StoredCompileInputs, -) +from sqlbuild.cli.compile_reuse._helpers.launch_records import write_launch_record +from sqlbuild.cli.compile_reuse._helpers.native_reuse import native_attempt +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -from sqlbuild.compiler.compile.constants import ( - COMPILE_CACHE_DISABLE_ENV_VAR, - COMPILE_CACHE_DISABLE_VALUE, -) -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.presentation.main.supports_color import supports_color def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: @@ -71,140 +18,9 @@ def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: project_dir: str = os.path.abspath( os.getcwd() if request.project_dir is None else request.project_dir ) - bypass: CompileReuseAttempt = CompileReuseAttempt( - outcome=CompileReuseOutcome.BYPASS, project_dir=Path(project_dir), started=started + attempt: CompileReuseAttempt = native_attempt( + request=request, project_dir=project_dir, started=started ) - if _bypasses_reuse(request=request, project_dir=project_dir): - return bypass - snapshot_ns: int = time.time_ns() - attempt: CompileReuseAttempt = replace( - bypass, - outcome=CompileReuseOutcome.MISS, - entry_path=compiler_cache_directory(Path(project_dir)) - / REUSE_ENTRY_DIRECTORY_NAME - / f"{entry_slot_name(selected_target=request.selected_target)}{REUSE_ENTRY_SUFFIX}", - runtime=runtime_identity(), - search_path=search_path_stamps(project_dir=project_dir), - invocation_digest=invocation_digest( - request=request, - project_dir=project_dir, - use_color=(not request.no_color) and supports_color(), - ), - snapshot=snapshot_project_files(project_dir=project_dir), - snapshot_ns=snapshot_ns, - ) - header: StoredCompileHeader | None = ( - None if attempt.entry_path is None else read_entry_header(path=attempt.entry_path) - ) - if header is not None: - attempt = _checked_attempt(attempt=attempt, inputs=header.inputs) - if attempt.outcome is CompileReuseOutcome.HIT: - return _replayed_attempt( - attempt=attempt, header=header, json_output=request.json_output - ) - attempt = replace( - attempt, - restamped=restamped_paths(stored=header.inputs.project_files, current=attempt.snapshot), - ) - return replace( - attempt, - digests=with_missing_digests( - project_dir=project_dir, - snapshot=attempt.snapshot, - digests=attempt.digests, - paths=frozenset(), - snapshot_ns=snapshot_ns, - ), - check_ms=elapsed_ms(started=started), - ) - - -def _bypasses_reuse(*, request: CompileReuseRequest, project_dir: str) -> bool: - return ( - request.no_cache - or request.manifest - or request.profiling - or request.debug - or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE - or os.environ.get(REUSE_DISABLE_ENV_VAR) == REUSE_DISABLE_VALUE - or not any( - os.path.isfile(os.path.join(project_dir, name)) for name in PROJECT_CONFIG_FILENAMES - ) - ) - - -def _checked_attempt( - *, attempt: CompileReuseAttempt, inputs: StoredCompileInputs -) -> CompileReuseAttempt: - project_dir: str = str(attempt.project_dir) - identity_unchanged: bool = ( - inputs.invocation_digest == attempt.invocation_digest - and inputs.runtime == attempt.runtime - and inputs.search_path == attempt.search_path - and tracked_environment_names(template_names=inputs.environment_names) - == inputs.environment_names - and environment_digest(names=inputs.environment_names) == inputs.environment_digest - and settings_inputs_digest(inputs=inputs.settings_inputs) == inputs.settings_digest - and module_stamps_unchanged(stamps=inputs.modules) - ) - comparison: ProjectFilesComparison = ( - compare_project_files( - project_dir=project_dir, stored=inputs.project_files, current=attempt.snapshot - ) - if identity_unchanged - else ProjectFilesComparison(unchanged=False, verified={}) - ) - unchanged: bool = comparison.unchanged and target_files_unchanged( - stored=inputs.target_files, project_dir=project_dir, tree=inputs.target_tree - ) - return replace( - attempt, - outcome=CompileReuseOutcome.HIT if unchanged else CompileReuseOutcome.MISS, - digests=carried_forward_digests( - stored=inputs.project_files, current=attempt.snapshot, verified=comparison.verified - ), - ) - - -def _replayed_attempt( - *, attempt: CompileReuseAttempt, header: StoredCompileHeader, json_output: bool -) -> CompileReuseAttempt: - stdout: str | None = ( - None - if attempt.entry_path is None - else read_entry_stdout(path=attempt.entry_path, header=header) - ) - span: tuple[int, int] | None = header.output.timings_span - if stdout is None or (json_output and span is None): - return replace(attempt, outcome=CompileReuseOutcome.MISS) - check_ms: int = elapsed_ms(started=attempt.started) - if span is not None: - stdout = replace_compile_timings( - stdout=stdout, - span=span, - timings={ - **reuse_timings(outcome=CompileReuseOutcome.HIT, check_ms=check_ms), - TOTAL_TIMING: elapsed_ms(started=attempt.started), - }, - ) - print(REUSE_HIT_MESSAGE.format(seconds=check_ms / MILLISECONDS_PER_SECOND), file=sys.stderr) - for line in header.output.stderr_lines: - print(line, file=sys.stderr) - _ = sys.stdout.write(stdout) - sys.stdout.flush() - if attempt.entry_path is not None and needs_refresh( - stored=header.inputs.project_files, current=attempt.snapshot - ): - rewrite_entry_inputs( - path=attempt.entry_path, - header=header, - inputs=replace( - header.inputs, - project_files=stored_project_files( - snapshot=attempt.snapshot, - digests=attempt.digests, - snapshot_ns=attempt.snapshot_ns, - ), - ), - ) - return replace(attempt, check_ms=check_ms, exit_code=header.output.exit_code) + if attempt.outcome is CompileReuseOutcome.HIT: + write_launch_record(attempt=attempt) + return attempt diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py deleted file mode 100644 index 9ffe463834..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py +++ /dev/null @@ -1,366 +0,0 @@ -"""Checksummed, atomically replaced storage for one reusable compile result.""" - -from __future__ import annotations - -import contextlib -import json -import os -import tempfile -import uuid -import zlib -from dataclasses import replace -from pathlib import Path -from typing import BinaryIO, cast - -from sqlbuild.cli.compile_reuse.constants import ( - REUSE_ENTRY_BYTE_ORDER, - REUSE_ENTRY_CHECKSUM_BYTES, - REUSE_ENTRY_LENGTH_BYTES, - REUSE_ENTRY_MAGIC, - REUSE_ENTRY_SUFFIX, - REUSE_FORMAT_VERSION, - REUSE_MAX_ENTRY_BYTES, - REUSE_MAX_STORED_ENTRIES, - REUSE_RETIRED_RENDER_SUFFIX, - REUSE_STDOUT_SEPARATOR, - REUSE_STDOUT_SUFFIX, -) -from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError -from sqlbuild.cli.compile_reuse.models import ( - SettingsEnvironmentInputs, - StoredCompileHeader, - StoredCompileInputs, - StoredCompileOutput, - StoredProjectFile, -) -from sqlbuild.cli.compile_reuse.types import FileStamp - -_TIMINGS_SPAN_LENGTH: int = 2 -_STAMP_FIELD_COUNT: int = 6 -_PROJECT_FILE_FIELD_COUNT: int = 8 -_SEARCH_PATH_FIELD_COUNT: int = 2 -_MODULE_FIELD_COUNT: int = 3 - - -def read_entry_header(*, path: Path) -> StoredCompileHeader | None: - """Read and verify the stored inputs, treating every fault as a cache miss.""" - - try: - with open(path, "rb") as handle: - if handle.read(len(REUSE_ENTRY_MAGIC)) != REUSE_ENTRY_MAGIC: - return None - metadata: bytes = _read_section(handle=handle) - if handle.read(1): - return None - payload: object = json.loads(metadata) - return _header(payload=payload) - except (OSError, UnicodeError, ValueError, TypeError, KeyError, IndexError): - return None - - -def read_entry_stdout(*, path: Path, header: StoredCompileHeader) -> str | None: - """Read and verify the stored stdout once every input matched.""" - - stdout_path: Path = path.parent / header.output.stdout_file - try: - stdout: bytes = stdout_path.read_bytes() - if ( - len(stdout) > REUSE_MAX_ENTRY_BYTES - or zlib.crc32(stdout) != header.output.stdout_checksum - ): - return None - text: str = stdout.decode("utf-8", "surrogateescape") - except (OSError, UnicodeError, ValueError): - return None - return text if len(text) == header.output.stdout_length else None - - -def write_entry( - *, - path: Path, - inputs: StoredCompileInputs, - output: StoredCompileOutput, - stdout: str, -) -> None: - """Publish one entry and its stdout atomically, filling in the stdout file and checksum.""" - - encoded_stdout: bytes = stdout.encode("utf-8", "surrogateescape") - stdout_file: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}{REUSE_STDOUT_SUFFIX}" - stored_output: StoredCompileOutput = replace( - output, stdout_file=stdout_file, stdout_checksum=zlib.crc32(encoded_stdout) - ) - metadata: bytes = _metadata(inputs=inputs, output=stored_output) - if len(metadata) + len(encoded_stdout) > REUSE_MAX_ENTRY_BYTES: - remove_entry(path=path) - return - try: - path.parent.mkdir(parents=True, exist_ok=True) - _publish_file(path=path.parent / stdout_file, contents=encoded_stdout) - _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) - _remove_side_files(entry_path=path, keep=stdout_file) - _prune_entries(directory=path.parent, keep=path) - except OSError: - remove_entry(path=path) - - -def rewrite_entry_inputs( - *, path: Path, header: StoredCompileHeader, inputs: StoredCompileInputs -) -> None: - """Replace only the stored inputs of an entry, keeping its stored stdout file.""" - - metadata: bytes = _metadata(inputs=inputs, output=header.output) - with contextlib.suppress(OSError): - _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) - - -def remove_entry(*, path: Path) -> None: - """Remove one stored entry and its stored stdout when they exist.""" - - with contextlib.suppress(OSError): - path.unlink(missing_ok=True) - _remove_side_files(entry_path=path, keep=None) - - -def _publish_file(*, path: Path, contents: bytes) -> None: - descriptor, temporary_path = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=".tmp", dir=path.parent - ) - try: - with os.fdopen(descriptor, "wb") as handle: - _ = handle.write(contents) - os.replace(temporary_path, path) - except BaseException: - with contextlib.suppress(OSError): - os.unlink(temporary_path) - raise - - -def _metadata(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> bytes: - return json.dumps(_payload(inputs=inputs, output=output), separators=(",", ":")).encode( - "utf-8", "surrogateescape" - ) - - -def _framed_metadata(*, metadata: bytes) -> bytes: - return b"".join( - ( - REUSE_ENTRY_MAGIC, - len(metadata).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), - zlib.crc32(metadata).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), - metadata, - ) - ) - - -def _remove_side_files(*, entry_path: Path, keep: str | None) -> None: - """Remove every file stored beside an entry except keep, including older releases' files.""" - - prefix: str = f"{entry_path.stem}{REUSE_STDOUT_SEPARATOR}" - try: - candidates: list[Path] = list(entry_path.parent.iterdir()) - except OSError: - return - for candidate in candidates: - if candidate.name.startswith(prefix) and candidate.name != keep: - with contextlib.suppress(OSError): - candidate.unlink() - - -def _read_section(*, handle: BinaryIO) -> bytes: - length: int = int.from_bytes(handle.read(REUSE_ENTRY_LENGTH_BYTES), REUSE_ENTRY_BYTE_ORDER) - checksum: int = int.from_bytes(handle.read(REUSE_ENTRY_CHECKSUM_BYTES), REUSE_ENTRY_BYTE_ORDER) - if length > REUSE_MAX_ENTRY_BYTES: - raise CompileReuseEntryError("stored compile section is too large") - data: bytes = handle.read(length) - if len(data) != length or zlib.crc32(data) != checksum: - raise CompileReuseEntryError("stored compile section is truncated or corrupt") - return data - - -def _prune_entries(*, directory: Path, keep: Path) -> None: - entries: list[tuple[int, Path]] = [] - for candidate in directory.iterdir(): - if candidate.suffix == REUSE_RETIRED_RENDER_SUFFIX: - with contextlib.suppress(OSError): - candidate.unlink() - continue - if candidate.suffix != REUSE_ENTRY_SUFFIX or candidate == keep: - continue - with contextlib.suppress(OSError): - entries.append((candidate.stat().st_mtime_ns, candidate)) - entries.sort(reverse=True) - for _, stale in entries[REUSE_MAX_STORED_ENTRIES - 1 :]: - remove_entry(path=stale) - - -def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dict[str, object]: - return { - "format": REUSE_FORMAT_VERSION, - "invocation_digest": inputs.invocation_digest, - "runtime": inputs.runtime, - "environment_names": list(inputs.environment_names), - "environment_digest": inputs.environment_digest, - "search_path": [list(item) for item in inputs.search_path], - "modules": [list(item) for item in inputs.modules], - "project_files": { - path: [*item.stamp, item.digest, item.racy] - for path, item in inputs.project_files.items() - }, - "target_files": {path: list(stamp) for path, stamp in inputs.target_files.items()}, - "stderr_lines": list(output.stderr_lines), - "exit_code": output.exit_code, - "timings_span": None if output.timings_span is None else list(output.timings_span), - "stdout_length": output.stdout_length, - "stdout_checksum": output.stdout_checksum, - "stdout_file": output.stdout_file, - "target_tree": inputs.target_tree, - "settings_inputs": [ - { - "case_sensitive": item.case_sensitive, - "names": list(item.names), - "prefixes": list(item.prefixes), - "env_files": list(item.env_files), - "secrets_dirs": list(item.secrets_dirs), - } - for item in inputs.settings_inputs - ], - "settings_digest": inputs.settings_digest, - } - - -def _header(*, payload: object) -> StoredCompileHeader: - fields: dict[str, object] = _mapping(payload) - if fields.get("format") != REUSE_FORMAT_VERSION: - raise CompileReuseEntryError("unknown stored compile format") - runtime: dict[str, object] = _mapping(fields["runtime"]) - span: tuple[int, int] | None = _timings_span(fields["timings_span"]) - return StoredCompileHeader( - inputs=StoredCompileInputs( - invocation_digest=_string(fields["invocation_digest"]), - runtime={_string(key): _string(value) for key, value in runtime.items()}, - environment_names=tuple(_string(name) for name in _list(fields["environment_names"])), - environment_digest=_string(fields["environment_digest"]), - search_path=tuple( - (_string(item[0]), _integer(item[1])) - for item in _rows(value=fields["search_path"], width=_SEARCH_PATH_FIELD_COUNT) - ), - modules=tuple( - (_string(item[0]), _integer(item[1]), _integer(item[2])) - for item in _rows(value=fields["modules"], width=_MODULE_FIELD_COUNT) - ), - project_files={ - _string(path): _project_file(item) - for path, item in _mapping(fields["project_files"]).items() - }, - target_files={ - _string(path): _stamp(item) - for path, item in _mapping(fields["target_files"]).items() - }, - target_tree=_boolean(fields["target_tree"]), - settings_inputs=tuple( - _settings_inputs(item) for item in _list(fields["settings_inputs"]) - ), - settings_digest=_string(fields["settings_digest"]), - ), - output=StoredCompileOutput( - stderr_lines=tuple(_string(line) for line in _list(fields["stderr_lines"])), - exit_code=_integer(fields["exit_code"]), - timings_span=span, - stdout_length=_integer(fields["stdout_length"]), - stdout_checksum=_integer(fields["stdout_checksum"]), - stdout_file=_stdout_file(fields["stdout_file"]), - ), - ) - - -def _settings_inputs(value: object) -> SettingsEnvironmentInputs: - fields: dict[str, object] = _mapping(value) - return SettingsEnvironmentInputs( - case_sensitive=_boolean(fields["case_sensitive"]), - names=_strings(fields["names"]), - prefixes=_strings(fields["prefixes"]), - env_files=_strings(fields["env_files"]), - secrets_dirs=_strings(fields["secrets_dirs"]), - ) - - -def _stdout_file(value: object) -> str: - name: str = _string(value) - if os.path.basename(name) != name or not name.endswith(REUSE_STDOUT_SUFFIX): - raise CompileReuseEntryError("invalid stored stdout file name") - return name - - -def _timings_span(value: object) -> tuple[int, int] | None: - if value is None: - return None - row: list[object] = _rows(value=[value], width=_TIMINGS_SPAN_LENGTH)[0] - return _integer(row[0]), _integer(row[1]) - - -def _project_file(value: object) -> StoredProjectFile: - if type(value) is not list or len(value) != _PROJECT_FILE_FIELD_COUNT: - raise CompileReuseEntryError("invalid stored project file") - row: list[object] = cast(list[object], value) - digest: object = row[6] - racy: object = row[7] - if type(racy) is not bool or not (digest is None or type(digest) is str): - raise CompileReuseEntryError("invalid stored project file") - return StoredProjectFile(stamp=_stamp(row[:_STAMP_FIELD_COUNT]), digest=digest, racy=racy) - - -def _stamp(value: object) -> FileStamp: - if type(value) is not list or len(value) != _STAMP_FIELD_COUNT: - raise CompileReuseEntryError("invalid stored file stamp") - kind, size, mtime_ns, ctime_ns, inode, link = cast(list[object], value) - if ( - type(kind) is not str - or type(size) is not int - or type(mtime_ns) is not int - or type(ctime_ns) is not int - or type(inode) is not int - or not (link is None or type(link) is str) - ): - raise CompileReuseEntryError("invalid stored file stamp") - return FileStamp(kind, size, mtime_ns, ctime_ns, inode, link) - - -def _rows(*, value: object, width: int) -> list[list[object]]: - rows: list[list[object]] = [_list(row) for row in _list(value)] - if any(len(row) != width for row in rows): - raise CompileReuseEntryError("invalid stored row") - return rows - - -def _list(value: object) -> list[object]: - if not isinstance(value, list): - raise CompileReuseEntryError("invalid stored list") - return cast(list[object], value) - - -def _mapping(value: object) -> dict[str, object]: - if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): - raise CompileReuseEntryError("invalid stored mapping") - return cast(dict[str, object], value) - - -def _strings(value: object) -> tuple[str, ...]: - return tuple(_string(item) for item in _list(value)) - - -def _boolean(value: object) -> bool: - if type(value) is not bool: - raise CompileReuseEntryError("invalid stored boolean") - return value - - -def _string(value: object) -> str: - if not isinstance(value, str): - raise CompileReuseEntryError("invalid stored string") - return value - - -def _integer(value: object) -> int: - if not isinstance(value, int) or isinstance(value, bool): - raise CompileReuseEntryError("invalid stored integer") - return value diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py b/src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py new file mode 100644 index 0000000000..1bbe417528 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/launch_records.py @@ -0,0 +1,121 @@ +"""Leave the native `sqb` executable what it needs to serve the next unchanged compile itself.""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import sys +import uuid +from pathlib import Path + +from sqlbuild._native import NativeReuseAttempt +from sqlbuild.cli.compile_reuse._helpers.native_reuse import reuse_rules +from sqlbuild.cli.compile_reuse._helpers.startup_inputs import startup_inputs +from sqlbuild.cli.compile_reuse.constants import ( + COMPILE_COMMAND_NAME, + COMPUTE_LOG_DIRECTORY_NAME, + LAUNCH_RECORD_DIRECTORY, + LAUNCH_RECORD_FORMAT, + LAUNCH_RECORD_LIMIT, + REUSE_BYPASS_TIMING, + REUSE_CHECK_TIMING, + REUSE_HIT_MESSAGE, + REUSE_HIT_TIMING, + REUSE_LOGGER_NAME, + REUSE_MISS_TIMING, + TOTAL_TIMING, +) +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.cli.entry.main._launch_record_path import launch_record_path +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR +from sqlbuild.runtime.compute_logs.classes.text_tee import TextComputeLogTee +from sqlbuild.runtime.output_capture.constants import INVOCATION_CONTEXT_ENV + +_LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) +_SECONDS_FIELD: str = "{seconds:.1f}" +_OUTPUT_FILES_RULE: str = "output_files" + + +def write_launch_record(*, attempt: CompileReuseAttempt) -> None: + """Record this reusable compile for `sqb` when nothing it cannot replay took part.""" + + path: Path | None = launch_record_path() + native: NativeReuseAttempt | None = attempt.native + if path is None or native is None or not _replayable_natively(): + return + directory: Path = attempt.project_dir.joinpath(*LAUNCH_RECORD_DIRECTORY) + if path.parent.resolve() != directory.resolve(): + return + try: + _write(path=path, payload=_payload(attempt=attempt, native=native, key=path.stem)) + _prune(directory=directory) + except OSError: + _LOGGER.debug("Compile reuse did not leave a launch record", exc_info=True) + + +def _replayable_natively() -> bool: + return INVOCATION_CONTEXT_ENV not in os.environ and not os.environ.get( + ANALYSIS_RECORD_DIR_ENV_VAR + ) + + +def _payload( + *, attempt: CompileReuseAttempt, native: NativeReuseAttempt, key: str +) -> dict[str, object]: + invocation_digest, runtime, search_path = native.launch_identity() + rules: dict[str, object] = { + name: value for name, value in reuse_rules().items() if name != _OUTPUT_FILES_RULE + } + project_dir: Path = attempt.project_dir.resolve() + prefix, suffix = REUSE_HIT_MESSAGE.split(_SECONDS_FIELD) + return { + "format": LAUNCH_RECORD_FORMAT, + "key": key, + "project_dir": str(attempt.project_dir), + "store_path": native.store_path(), + "invocation_digest": invocation_digest, + "runtime": runtime, + "search_path": search_path, + "startup_inputs": startup_inputs(), + "json_output": attempt.json_output, + "rules": rules, + "capture": { + "root": str((project_dir / COMPUTE_LOG_DIRECTORY_NAME).resolve()), + "command": COMPILE_COMMAND_NAME, + "project_dir": str(project_dir), + "target": attempt.selected_target, + } + if isinstance(sys.stdout, TextComputeLogTee) + else None, + "hit_message": [prefix, suffix], + "timings": { + "hit": REUSE_HIT_TIMING, + "miss": REUSE_MISS_TIMING, + "bypass": REUSE_BYPASS_TIMING, + "check": REUSE_CHECK_TIMING, + "total": TOTAL_TIMING, + }, + } + + +def _write(*, path: Path, payload: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary: Path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + _ = temporary.write_text(json.dumps(payload), encoding="utf-8") + os.replace(temporary, path) + finally: + with contextlib.suppress(OSError): + temporary.unlink(missing_ok=True) + + +def _prune(*, directory: Path) -> None: + records: list[tuple[int, Path]] = [ + (entry.stat().st_mtime_ns, entry) for entry in directory.glob("*.json") + ] + records.sort(reverse=True) + for _, stale in records[LAUNCH_RECORD_LIMIT:]: + with contextlib.suppress(OSError): + stale.unlink() diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py b/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py new file mode 100644 index 0000000000..05d68358be --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py @@ -0,0 +1,322 @@ +"""Check, replay and record whole-project reuse through the native store.""" + +from __future__ import annotations + +import contextlib +import logging +import os +import sys +import time +from dataclasses import replace +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild._native import NativeReuseAttempt +from sqlbuild.cli.compile_reuse._helpers.provider_settings import provider_settings_inputs +from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( + invocation_identity, + redirected_output_files, + runtime_identity, +) +from sqlbuild.cli.compile_reuse._helpers.timings import elapsed_ms, reuse_timings +from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites +from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput +from sqlbuild.cli.compile_reuse.constants import ( + BYTES_PER_MEBIBYTE, + COMPILED_DIRECTORY_NAME, + EXCLUDED_DIRECTORIES, + EXCLUDED_ROOT_DIRECTORIES, + MILLISECONDS_PER_SECOND, + MISSING_ENVIRONMENT_VALUE, + MISSING_FILE_DIGEST, + MISSING_PATH_MTIME_NS, + NATIVE_REUSE_DIRECTORY_NAME, + PRESENCE_ONLY_FILE_SUFFIXES, + PROJECT_CONFIG_FILENAMES, + PROJECT_ROOT_PATH_MTIME_NS, + RACY_WINDOW_NS, + RETIRED_REUSE_DIRECTORY_NAME, + REUSE_DISABLE_ENV_VAR, + REUSE_DISABLE_VALUE, + REUSE_HIT_MESSAGE, + REUSE_LOGGER_NAME, + REUSE_MAX_ENTRY_BYTES, + REUSE_MAX_STORED_ENTRIES, + REUSE_STORE_DONE_MESSAGE, + REUSE_STORE_NOTICE_BYTES, + REUSE_STORE_NOTICE_PATHS, + REUSE_STORE_SKIPPED_MESSAGE, + REUSE_STORE_START_MESSAGE, + TARGET_DIRECTORY_NAME, + TOTAL_TIMING, + TRACKED_ENVIRONMENT_PREFIXES, + UNTRACKED_ENVIRONMENT_NAMES, +) +from sqlbuild.cli.compile_reuse.models import ( + CompileReuseAttempt, + CompileReuseRequest, + SettingsInputsResult, +) +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome +from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads +from sqlbuild.compiler.compile.constants import ( + COMPILE_CACHE_DISABLE_ENV_VAR, + COMPILE_CACHE_DISABLE_VALUE, +) +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.macro_bridge.main.macro_store_module_paths import ( + macro_store_module_paths, +) +from sqlbuild.presentation.main.supports_color import supports_color + +_LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) +_OUTCOMES: dict[str, CompileReuseOutcome] = { + outcome.value: outcome for outcome in CompileReuseOutcome +} +_STORED: str = "stored" +_SKIPPED: str = "skipped" + + +def native_attempt( + *, request: CompileReuseRequest, project_dir: str, started: float +) -> CompileReuseAttempt: + """Check reuse natively, replaying on a hit.""" + + attempt: NativeReuseAttempt = _native.check_compile_reuse( + { + "project_dir": project_dir, + "store_directory": compiler_cache_directory(Path(project_dir)) + / NATIVE_REUSE_DIRECTORY_NAME, + "selected_target": request.selected_target, + "invocation": invocation_identity( + request=request, + project_dir=project_dir, + use_color=(not request.no_color) and supports_color(), + ), + "runtime": runtime_identity(), + "search_path": list(sys.path), + "bypass_requested": request.no_cache + or request.manifest + or request.profiling + or request.debug, + "json_output": request.json_output, + "rules": reuse_rules(), + } + ) + outcome: CompileReuseOutcome = _OUTCOMES[attempt.outcome()] + base: CompileReuseAttempt = CompileReuseAttempt( + outcome=outcome, + project_dir=Path(project_dir), + started=started, + selected_target=request.selected_target, + json_output=request.json_output, + ) + if outcome is CompileReuseOutcome.BYPASS: + return base + report_native_answer( + stage=NativeStage.COMPILE_OUTPUTS, + kind="reuse_snapshot_paths", + units=attempt.snapshot_paths(), + ) + report_native_answer( + stage=NativeStage.COMPILE_OUTPUTS, + kind="reuse_digested_files", + units=attempt.digested_files(), + ) + missed: CompileReuseAttempt = replace( + base, + outcome=CompileReuseOutcome.MISS, + entry_path=Path(attempt.store_path()), + native=attempt, + check_ms=0 if attempt.replay_failed() else elapsed_ms(started=started), + ) + if outcome is CompileReuseOutcome.HIT: + return _replayed(attempt=attempt, missed=missed, started=started, json=request.json_output) + return missed + + +def _replayed( + *, + attempt: NativeReuseAttempt, + missed: CompileReuseAttempt, + started: float, + json: bool, +) -> CompileReuseAttempt: + check_ms: int = elapsed_ms(started=started) + timings: dict[str, int] = { + **reuse_timings(outcome=CompileReuseOutcome.HIT, check_ms=check_ms), + TOTAL_TIMING: elapsed_ms(started=started), + } + replay: tuple[str, list[str], int] | None = attempt.replay( + list(timings.items()) if json else [] + ) + if replay is None: + return replace(missed, check_ms=0) + stdout, stderr_lines, exit_code = replay + print(REUSE_HIT_MESSAGE.format(seconds=check_ms / MILLISECONDS_PER_SECOND), file=sys.stderr) + for line in stderr_lines: + print(line, file=sys.stderr) + _ = sys.stdout.write(stdout) + sys.stdout.flush() + report_native_answer(stage=NativeStage.COMPILE_OUTPUTS, kind="reuse_replays") + return replace(missed, outcome=CompileReuseOutcome.HIT, check_ms=check_ms, exit_code=exit_code) + + +def record_native( + *, + store_path: Path, + native: NativeReuseAttempt, + output: RecordedCompileOutput, + exit_code: int, + input_reads: CompileInputReads, + artifact_writes: CompileArtifactWrites, + compile_cache_enabled: bool, + artifacts_written: bool, + dag_artifact_path: Path | None, + json_output: bool, +) -> bool: + """Store a reusable compile natively, or clear the slot; return whether it was stored.""" + + try: + return _record( + native=native, + output=output, + exit_code=exit_code, + input_reads=input_reads, + artifact_writes=artifact_writes, + compile_cache_enabled=compile_cache_enabled, + artifacts_written=artifacts_written, + dag_artifact_path=dag_artifact_path, + json_output=json_output, + ) + except Exception: + _LOGGER.debug("Compile reuse did not store this compile", exc_info=True) + with contextlib.suppress(OSError): + store_path.unlink(missing_ok=True) + return False + + +def _record( + *, + native: NativeReuseAttempt, + output: RecordedCompileOutput, + exit_code: int, + input_reads: CompileInputReads, + artifact_writes: CompileArtifactWrites, + compile_cache_enabled: bool, + artifacts_written: bool, + dag_artifact_path: Path | None, + json_output: bool, +) -> bool: + settings: SettingsInputsResult = provider_settings_inputs( + settings_classes=input_reads.settings_classes + ) + if settings.unsupported_reason is not None: + _LOGGER.debug("Compile reuse is off for this project: %s", settings.unsupported_reason) + eligible: bool = ( + output.stdout is not None + and compile_cache_enabled + and not input_reads.read_run_id + and settings.unsupported_reason is None + ) + notice_started: float | None = _start_notice(native=native) if eligible else None + outcome, digested_files = native.record( + { + "stdout": output.stdout, + "stderr_lines": list(output.stderr_lines), + "exit_code": exit_code, + "compile_cache_enabled": compile_cache_enabled, + "read_run_id": input_reads.read_run_id, + "settings_inputs": None + if settings.unsupported_reason is not None + else [ + ( + item.case_sensitive, + list(item.names), + list(item.prefixes), + list(item.env_files), + list(item.secrets_dirs), + ) + for item in settings.inputs + ], + "template_environment_names": list(input_reads.environment_names), + "module_paths": _module_paths(), + "artifacts": { + path: (artifact.digest, artifact.size, artifact.mtime_ns) + for path, artifact in artifact_writes.artifacts.items() + }, + "artifacts_written": artifacts_written, + "dag_path": None if dag_artifact_path is None else os.path.abspath(dag_artifact_path), + "json_output": json_output, + "rules": reuse_rules(), + } + ) + report_native_answer( + stage=NativeStage.COMPILE_OUTPUTS, kind="reuse_digested_files", units=digested_files + ) + if outcome == _STORED: + report_native_answer(stage=NativeStage.COMPILE_OUTPUTS, kind="reuse_records") + _finish_notice( + started=notice_started, + message=REUSE_STORE_SKIPPED_MESSAGE if outcome == _SKIPPED else REUSE_STORE_DONE_MESSAGE, + ) + return outcome == _STORED + + +def reuse_rules() -> dict[str, object]: + """The reuse constants the native check and record share with the Python implementation.""" + + return { + "excluded": sorted(EXCLUDED_DIRECTORIES), + "excluded_root": sorted(EXCLUDED_ROOT_DIRECTORIES), + "presence_suffixes": list(PRESENCE_ONLY_FILE_SUFFIXES), + "output_files": redirected_output_files(), + "tracked_prefixes": list(TRACKED_ENVIRONMENT_PREFIXES), + "untracked_names": sorted(UNTRACKED_ENVIRONMENT_NAMES), + "disabling_variables": [ + (COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE), + (REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE), + ], + "config_filenames": list(PROJECT_CONFIG_FILENAMES), + "racy_window_ns": RACY_WINDOW_NS, + "max_stored_entries": REUSE_MAX_STORED_ENTRIES, + "max_entry_bytes": REUSE_MAX_ENTRY_BYTES, + "missing_environment_value": MISSING_ENVIRONMENT_VALUE, + "missing_file_digest": MISSING_FILE_DIGEST, + "missing_path_mtime_ns": MISSING_PATH_MTIME_NS, + "project_root_path_mtime_ns": PROJECT_ROOT_PATH_MTIME_NS, + "compiled_root": os.path.join(TARGET_DIRECTORY_NAME, COMPILED_DIRECTORY_NAME), + "retired_directories": [RETIRED_REUSE_DIRECTORY_NAME], + } + + +def _module_paths() -> list[str]: + """Loaded module files, then the macro store's extra module paths, as Python lists them.""" + + module_files: list[object] = [ + getattr(module, "__file__", None) for module in tuple(sys.modules.values()) + ] + return [ + path + for path in (*module_files, *sorted(macro_store_module_paths())) + if isinstance(path, str) + ] + + +def _start_notice(*, native: NativeReuseAttempt) -> float | None: + pending_bytes: int = native.pending_digest_bytes() + paths: int = native.snapshot_paths() + if pending_bytes < REUSE_STORE_NOTICE_BYTES and paths < REUSE_STORE_NOTICE_PATHS: + return None + print( + REUSE_STORE_START_MESSAGE.format(paths=paths, mebibytes=pending_bytes / BYTES_PER_MEBIBYTE), + file=sys.stderr, + ) + return time.monotonic() + + +def _finish_notice(*, started: float | None, message: str) -> None: + if started is not None: + print(message.format(seconds=time.monotonic() - started), file=sys.stderr) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py deleted file mode 100644 index 5c0472b33c..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py +++ /dev/null @@ -1,286 +0,0 @@ -"""Stat walk, content digests, and change detection for project input files.""" - -from __future__ import annotations - -import hashlib -import os -import stat - -from sqlbuild.cli.compile_reuse.constants import ( - BROKEN_LINK_KIND, - DIGEST_SIZE_BYTES, - DIRECTORY_KIND, - DIRECTORY_LINK_KIND, - EXCLUDED_DIRECTORIES, - EXCLUDED_ROOT_DIRECTORIES, - FILE_KIND, - FILE_LINK_KIND, - OUTPUT_FILE_DESCRIPTORS, - PRESENCE_ONLY_FILE_SUFFIXES, - PRESENCE_ONLY_KIND, - RACY_WINDOW_NS, - READ_CHUNK_BYTES, - SPECIAL_FILE_KIND, -) -from sqlbuild.cli.compile_reuse.models import ProjectFilesComparison, StoredProjectFile -from sqlbuild.cli.compile_reuse.types import FileStamp - -_DIRECTORY_STAMP: FileStamp = FileStamp(DIRECTORY_KIND, 0, 0, 0, 0, None) -_PRESENCE_STAMP: FileStamp = FileStamp(PRESENCE_ONLY_KIND, 0, 0, 0, 0, None) -_PRESENCE_ONLY_KINDS: frozenset[str] = frozenset( - {DIRECTORY_KIND, DIRECTORY_LINK_KIND, BROKEN_LINK_KIND, PRESENCE_ONLY_KIND} -) -_HASHABLE_KINDS: frozenset[str] = frozenset({FILE_KIND, FILE_LINK_KIND}) - - -def snapshot_project_files(*, project_dir: str) -> dict[str, FileStamp]: - """Stat every compile-relevant path under the project, keyed by relative path.""" - - stamps: dict[str, FileStamp] = {} - prefix_length: int = len(project_dir) + 1 - output_files: frozenset[tuple[int, int]] = _redirected_output_files() - visited_links: set[str] = {os.path.realpath(project_dir)} - pending: list[tuple[str, bool]] = [(project_dir, True)] - while pending: - directory, is_root = pending.pop() - for entry in _directory_entries(directory=directory): - relative_path: str = entry.path[prefix_length:] - stamp: FileStamp | None = _entry_stamp( - entry=entry, is_root=is_root, output_files=output_files - ) - if stamp is None: - continue - stamps[relative_path] = stamp - if stamp.kind == DIRECTORY_KIND: - pending.append((entry.path, False)) - elif stamp.kind == DIRECTORY_LINK_KIND: - resolved: str = os.path.realpath(entry.path) - if resolved not in visited_links: - visited_links.add(resolved) - pending.append((entry.path, False)) - return stamps - - -def file_digest(*, path: str) -> str | None: - """Return the content digest of one file, or None when it cannot be read.""" - - digest: hashlib.blake2b = hashlib.blake2b(digest_size=DIGEST_SIZE_BYTES) - try: - with open(path, "rb") as handle: - while chunk := handle.read(READ_CHUNK_BYTES): - digest.update(chunk) - except OSError: - return None - return digest.hexdigest() - - -def compare_project_files( - *, - project_dir: str, - stored: dict[str, StoredProjectFile], - current: dict[str, FileStamp], -) -> ProjectFilesComparison: - """Compare stat identities, hashing content only where stat changed or may be racy.""" - - verified: dict[str, str] = {} - changed: ProjectFilesComparison = ProjectFilesComparison(unchanged=False, verified=verified) - if len(stored) != len(current): - return changed - for relative_path, stamp in current.items(): - previous: StoredProjectFile | None = stored.get(relative_path) - if previous is None or not _same_identity(stamp=stamp, recorded=previous.stamp): - return changed - if stamp.kind in _PRESENCE_ONLY_KINDS or (stamp == previous.stamp and not previous.racy): - continue - if stamp.kind not in _HASHABLE_KINDS or previous.digest is None: - return changed - digest: str | None = file_digest(path=os.path.join(project_dir, relative_path)) - if digest is None or digest != previous.digest: - return changed - verified[relative_path] = digest - return ProjectFilesComparison(unchanged=True, verified=verified) - - -def carried_forward_digests( - *, - stored: dict[str, StoredProjectFile], - current: dict[str, FileStamp], - verified: dict[str, str], -) -> dict[str, str]: - """Return digests still valid for current files, without reading any content.""" - - digests: dict[str, str] = dict(verified) - for relative_path, stamp in current.items(): - previous: StoredProjectFile | None = stored.get(relative_path) - if ( - relative_path not in digests - and previous is not None - and previous.digest is not None - and not previous.racy - and previous.stamp == stamp - ): - digests[relative_path] = previous.digest - return digests - - -def restamped_paths( - *, stored: dict[str, StoredProjectFile], current: dict[str, FileStamp] -) -> frozenset[str]: - """Return hashable files whose stat identity moved since the stored compile.""" - - return frozenset( - relative_path - for relative_path, stamp in current.items() - if stamp.kind in _HASHABLE_KINDS - and (previous := stored.get(relative_path)) is not None - and previous.stamp.kind == stamp.kind - and previous.stamp != stamp - ) - - -def stored_project_files( - *, snapshot: dict[str, FileStamp], digests: dict[str, str], snapshot_ns: int -) -> dict[str, StoredProjectFile]: - """Pair each project path's stat identity with any digest already known for it.""" - - return { - relative_path: StoredProjectFile( - stamp=stamp, - digest=digests.get(relative_path), - racy=is_racy(stamp=stamp, snapshot_ns=snapshot_ns), - ) - for relative_path, stamp in snapshot.items() - } - - -def needs_refresh(*, stored: dict[str, StoredProjectFile], current: dict[str, FileStamp]) -> bool: - """Return whether a hit verified any file by content, so its new stamp is worth storing.""" - - return any( - (previous := stored.get(relative_path)) is not None - and (previous.racy or previous.stamp != stamp) - and stamp.kind in _HASHABLE_KINDS - for relative_path, stamp in current.items() - ) - - -def is_racy(*, stamp: FileStamp, snapshot_ns: int) -> bool: - """Return whether a same-tick edit after the snapshot could leave this stamp unchanged.""" - - return ( - stamp.kind in _HASHABLE_KINDS - and max(stamp.mtime_ns, stamp.ctime_ns) >= snapshot_ns - RACY_WINDOW_NS - ) - - -def with_missing_digests( - *, - project_dir: str, - snapshot: dict[str, FileStamp], - digests: dict[str, str], - paths: frozenset[str], - snapshot_ns: int, -) -> dict[str, str]: - """Return digests extended with racy files, plus the given paths, that still lack one.""" - - completed: dict[str, str] = dict(digests) - for relative_path, stamp in snapshot.items(): - if ( - stamp.kind not in _HASHABLE_KINDS - or relative_path in completed - or not (relative_path in paths or is_racy(stamp=stamp, snapshot_ns=snapshot_ns)) - ): - continue - digest: str | None = file_digest(path=os.path.join(project_dir, relative_path)) - if digest is not None: - completed[relative_path] = digest - return completed - - -def pending_digest_bytes( - *, snapshot: dict[str, FileStamp], digests: dict[str, str], paths: frozenset[str] -) -> int: - """Return how many bytes hashing the given paths that still lack a digest would read.""" - - return sum( - snapshot[relative_path].size - for relative_path in paths - if relative_path in snapshot and relative_path not in digests - ) - - -def _same_identity(*, stamp: FileStamp, recorded: FileStamp) -> bool: - return ( - stamp.kind == recorded.kind - and stamp.link == recorded.link - and (stamp.kind in _PRESENCE_ONLY_KINDS or stamp.size == recorded.size) - ) - - -def _directory_entries(*, directory: str) -> list[os.DirEntry[str]]: - try: - with os.scandir(directory) as entries: - return list(entries) - except OSError: - return [] - - -def _entry_stamp( - *, entry: os.DirEntry[str], is_root: bool, output_files: frozenset[tuple[int, int]] -) -> FileStamp | None: - if entry.is_symlink(): - return _link_stamp(entry=entry, is_root=is_root) - if entry.is_dir(follow_symlinks=False): - return None if _excluded_directory(name=entry.name, is_root=is_root) else _DIRECTORY_STAMP - if entry.name.endswith(PRESENCE_ONLY_FILE_SUFFIXES): - return _PRESENCE_STAMP - try: - status: os.stat_result = entry.stat(follow_symlinks=False) - except OSError: - return None - if (status.st_dev, status.st_ino) in output_files: - return _PRESENCE_STAMP - return _file_stamp(status=status, link=None) - - -def _link_stamp(*, entry: os.DirEntry[str], is_root: bool) -> FileStamp | None: - try: - link: str = os.readlink(entry.path) - except OSError: - return None - try: - status: os.stat_result = os.stat(entry.path) - except OSError: - return FileStamp(BROKEN_LINK_KIND, 0, 0, 0, 0, link) - if not stat.S_ISDIR(status.st_mode): - return _file_stamp(status=status, link=link) - if _excluded_directory(name=entry.name, is_root=is_root): - return None - return FileStamp(DIRECTORY_LINK_KIND, 0, 0, 0, 0, link) - - -def _redirected_output_files() -> frozenset[tuple[int, int]]: - """Identify stdout and stderr files, which hold this command's output, not its inputs.""" - - identities: set[tuple[int, int]] = set() - for descriptor in OUTPUT_FILE_DESCRIPTORS: - try: - status: os.stat_result = os.fstat(descriptor) - except OSError: - continue - if stat.S_ISREG(status.st_mode): - identities.add((status.st_dev, status.st_ino)) - return frozenset(identities) - - -def _excluded_directory(*, name: str, is_root: bool) -> bool: - return name in EXCLUDED_DIRECTORIES or (is_root and name in EXCLUDED_ROOT_DIRECTORIES) - - -def _file_stamp(*, status: os.stat_result, link: str | None) -> FileStamp: - kind: str = SPECIAL_FILE_KIND - if stat.S_ISREG(status.st_mode): - kind = FILE_KIND if link is None else FILE_LINK_KIND - return FileStamp( - kind, status.st_size, status.st_mtime_ns, status.st_ctime_ns, status.st_ino, link - ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py index 44637f294a..0facf3a4f8 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py @@ -1,25 +1,15 @@ -"""Interpreter, installed code, environment, and invocation identity for compile reuse.""" +"""Interpreter and invocation facts the Python host supplies to native compile reuse.""" from __future__ import annotations -import hashlib import json import os +import stat import sys import sqlbuild._native as native -from sqlbuild.cli.compile_reuse._helpers.project_files import file_digest -from sqlbuild.cli.compile_reuse.constants import ( - DIGEST_SIZE_BYTES, - MISSING_ENVIRONMENT_VALUE, - MISSING_FILE_DIGEST, - MISSING_PATH_MTIME_NS, - PROJECT_ROOT_PATH_MTIME_NS, - REUSE_FORMAT_VERSION, - TRACKED_ENVIRONMENT_PREFIXES, - UNTRACKED_ENVIRONMENT_NAMES, -) -from sqlbuild.cli.compile_reuse.models import CompileReuseRequest, SettingsEnvironmentInputs +from sqlbuild.cli.compile_reuse.constants import OUTPUT_FILE_DESCRIPTORS, REUSE_FORMAT_VERSION +from sqlbuild.cli.compile_reuse.models import CompileReuseRequest from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine @@ -36,102 +26,10 @@ def runtime_identity() -> dict[str, str]: } -def search_path_stamps(*, project_dir: str) -> tuple[tuple[str, int], ...]: - """Stat import search path entries, leaving the project root to the project walk.""" - - project_root: str = os.path.realpath(project_dir) - stamps: list[tuple[str, int]] = [] - for entry in sys.path: - path: str = entry or "." - try: - mtime_ns: int = ( - PROJECT_ROOT_PATH_MTIME_NS - if os.path.realpath(path) == project_root - else os.stat(path).st_mtime_ns - ) - except OSError: - mtime_ns = MISSING_PATH_MTIME_NS - stamps.append((entry, mtime_ns)) - return tuple(stamps) - - -def loaded_module_stamps( - *, covered_paths: frozenset[str], extra_paths: frozenset[str] -) -> tuple[tuple[str, int, int], ...]: - """Stat every loaded module file, and `extra_paths`, that the project walk does not cover.""" - - stamps: dict[str, tuple[str, int, int]] = {} - module_paths: tuple[object, ...] = tuple( - getattr(module, "__file__", None) for module in tuple(sys.modules.values()) - ) - for path in (*module_paths, *sorted(extra_paths)): - if not isinstance(path, str) or path in stamps or path in covered_paths: - continue - try: - status: os.stat_result = os.stat(path) - except OSError: - continue - stamps[path] = (path, status.st_mtime_ns, status.st_size) - return tuple(sorted(stamps.values())) - - -def module_stamps_unchanged(*, stamps: tuple[tuple[str, int, int], ...]) -> bool: - """Return whether every recorded module file still has the same stat identity.""" - - for path, mtime_ns, size in stamps: - try: - status: os.stat_result = os.stat(path) - except OSError: - return False - if status.st_mtime_ns != mtime_ns or status.st_size != size: - return False - return True - - -def tracked_environment_names(*, template_names: tuple[str, ...]) -> tuple[str, ...]: - """Return tracked SQLBuild variables (not engine or capture settings) plus template reads.""" - - prefixed: set[str] = { - name - for name in os.environ - if name.startswith(TRACKED_ENVIRONMENT_PREFIXES) and name not in UNTRACKED_ENVIRONMENT_NAMES - } - return tuple(sorted(prefixed.union(template_names))) - - -def environment_digest(*, names: tuple[str, ...]) -> str: - """Hash the current values of environment variables without retaining them.""" - - digest: hashlib.blake2b = hashlib.blake2b(digest_size=DIGEST_SIZE_BYTES) - for name in names: - digest.update(name.encode("utf-8", "surrogateescape")) - digest.update(b"\0") - digest.update( - os.environ.get(name, MISSING_ENVIRONMENT_VALUE).encode("utf-8", "surrogateescape") - ) - digest.update(b"\0") - return digest.hexdigest() - - -def settings_inputs_digest(*, inputs: tuple[SettingsEnvironmentInputs, ...]) -> str: - """Hash the provider settings values in the environment, env files, and secrets dirs.""" - - fields: list[str] = [] - for item in inputs: - for name, value in _settings_environment(inputs=item): - fields.extend(("env", name, value)) - for path in item.env_files: - fields.extend(("env_file", path, _file_contents_digest(path))) - for directory in item.secrets_dirs: - fields.extend(("secrets_dir", directory)) - for relative_path, contents_digest in _directory_contents(directory=directory): - fields.extend((relative_path, contents_digest)) - encoded: bytes = "\0".join(fields).encode("utf-8", "surrogateescape") - return hashlib.blake2b(encoded, digest_size=DIGEST_SIZE_BYTES).hexdigest() - - -def invocation_digest(*, request: CompileReuseRequest, project_dir: str, use_color: bool) -> str: - """Hash every command-line choice and location that can change compile output.""" +def invocation_identity( + *, request: CompileReuseRequest, project_dir: str, use_color: bool +) -> bytes: + """Encode every command-line choice and location that can change compile output.""" payload: dict[str, object] = { "project_dir": project_dir, @@ -152,38 +50,20 @@ def invocation_digest(*, request: CompileReuseRequest, project_dir: str, use_col "sys_path": list(sys.path), "compiler_engine": resolve_compiler_engine().value, } - encoded: bytes = json.dumps( - payload, sort_keys=True, separators=(",", ":"), default=repr - ).encode("utf-8", "surrogateescape") - return hashlib.blake2b(encoded, digest_size=DIGEST_SIZE_BYTES).hexdigest() - - -def _settings_environment(*, inputs: SettingsEnvironmentInputs) -> list[tuple[str, str]]: - names: frozenset[str] = frozenset(inputs.names) - matched: list[tuple[str, str]] = [] - for name, value in os.environ.items(): - key: str = name if inputs.case_sensitive else name.lower() - if key in names or key.startswith(inputs.prefixes): - matched.append((name, value)) - return sorted(matched) - - -def _file_contents_digest(path: str) -> str: - return file_digest(path=path) or MISSING_FILE_DIGEST - - -def _directory_contents(*, directory: str) -> list[tuple[str, str]]: - contents: list[tuple[str, str]] = [] - for root, directories, filenames in os.walk(directory): - directories.sort() - for filename in sorted(filenames): - path: str = os.path.join(root, filename) - contents.append((os.path.relpath(path, directory), _file_contents_digest(path))) - return contents + return json.dumps(payload, sort_keys=True, separators=(",", ":"), default=repr).encode( + "utf-8", "surrogateescape" + ) -def entry_slot_name(*, selected_target: str | None) -> str: - """Name the single stored entry kept for one selected target.""" +def redirected_output_files() -> list[tuple[int, int]]: + """Identify stdout and stderr files, which hold this command's output, not its inputs.""" - label: str = "" if selected_target is None else f"target:{selected_target}" - return hashlib.blake2b(label.encode("utf-8"), digest_size=DIGEST_SIZE_BYTES).hexdigest() + identities: set[tuple[int, int]] = set() + for descriptor in OUTPUT_FILE_DESCRIPTORS: + try: + status: os.stat_result = os.fstat(descriptor) + except OSError: + continue + if stat.S_ISREG(status.st_mode): + identities.add((status.st_dev, status.st_ino)) + return sorted(identities) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py b/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py new file mode 100644 index 0000000000..0cc52f976d --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/startup_inputs.py @@ -0,0 +1,32 @@ +"""Interpreter and site-path inputs that must remain unchanged before skipping Python startup.""" + +import site +import sys +from os import stat_result +from pathlib import Path + + +def startup_inputs() -> list[tuple[str, int | None, int | None]]: + """Stamp startup directories, path files, interpreter and virtual-environment configuration.""" + + directories: set[Path] = { + Path(entry).absolute() + for entry in (*sys.path, *site.getsitepackages(), site.getusersitepackages()) + } + paths: set[Path] = { + *directories, + Path(sys.executable), + Path(sys.prefix) / "pyvenv.cfg", + Path(sys.base_prefix) / "pyvenv.cfg", + } + for directory in directories: + paths.update(directory.glob("*.pth")) + stamps: list[tuple[str, int | None, int | None]] = [] + for path in sorted(paths): + try: + status: stat_result = path.stat() + except FileNotFoundError: + stamps.append((str(path), None, None)) + else: + stamps.append((str(path), status.st_mtime_ns, status.st_size)) + return stamps diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/store.py b/src/sqlbuild/cli/compile_reuse/_helpers/store.py index 9a3a8552f0..8bbbacb5fe 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/store.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/store.py @@ -2,52 +2,15 @@ from __future__ import annotations -import logging -import os -import sys -import time from pathlib import Path -from sqlbuild.cli.compile_reuse._helpers.entry_file import remove_entry, write_entry -from sqlbuild.cli.compile_reuse._helpers.project_files import ( - pending_digest_bytes, - snapshot_project_files, - stored_project_files, - with_missing_digests, -) -from sqlbuild.cli.compile_reuse._helpers.provider_settings import provider_settings_inputs -from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( - environment_digest, - loaded_module_stamps, - settings_inputs_digest, - tracked_environment_names, -) -from sqlbuild.cli.compile_reuse._helpers.target_files import verified_target_files -from sqlbuild.cli.compile_reuse._helpers.timings import compile_timings_span +from sqlbuild.cli.compile_reuse._helpers.launch_records import write_launch_record +from sqlbuild.cli.compile_reuse._helpers.native_reuse import record_native from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput -from sqlbuild.cli.compile_reuse.constants import ( - BYTES_PER_MEBIBYTE, - REUSE_LOGGER_NAME, - REUSE_STORE_DONE_MESSAGE, - REUSE_STORE_NOTICE_BYTES, - REUSE_STORE_NOTICE_PATHS, - REUSE_STORE_SKIPPED_MESSAGE, - REUSE_STORE_START_MESSAGE, -) -from sqlbuild.cli.compile_reuse.models import ( - CompileReuseAttempt, - SettingsInputsResult, - StoredCompileInputs, - StoredCompileOutput, -) -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome, FileStamp +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads -from sqlbuild.compiler.macro_bridge.main.macro_store_module_paths import ( - macro_store_module_paths, -) - -_LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) def write_compile_entry( @@ -64,132 +27,23 @@ def write_compile_entry( ) -> None: """Store a reusable compile, or drop the stored one when this compile cannot be reused.""" - if attempt.outcome is not CompileReuseOutcome.MISS or attempt.entry_path is None: - return - try: - _write_compile_entry( - attempt=attempt, - entry_path=attempt.entry_path, - output=output, - exit_code=exit_code, - input_reads=input_reads, - artifact_writes=artifact_writes, - compile_cache_enabled=compile_cache_enabled, - artifacts_written=artifacts_written, - dag_artifact_path=dag_artifact_path, - json_output=json_output, - ) - except Exception: - _LOGGER.debug("Compile reuse did not store this compile", exc_info=True) - remove_entry(path=attempt.entry_path) - - -def _write_compile_entry( - *, - attempt: CompileReuseAttempt, - entry_path: Path, - output: RecordedCompileOutput, - exit_code: int, - input_reads: CompileInputReads, - artifact_writes: CompileArtifactWrites, - compile_cache_enabled: bool, - artifacts_written: bool, - dag_artifact_path: Path | None, - json_output: bool, -) -> None: - stdout: str | None = output.stdout - settings: SettingsInputsResult = provider_settings_inputs( - settings_classes=input_reads.settings_classes - ) - if settings.unsupported_reason is not None: - _LOGGER.debug("Compile reuse is off for this project: %s", settings.unsupported_reason) if ( - stdout is None - or not compile_cache_enabled - or input_reads.read_run_id - or settings.unsupported_reason is not None + attempt.outcome is not CompileReuseOutcome.MISS + or attempt.entry_path is None + or attempt.native is None ): - remove_entry(path=entry_path) return - project_dir: str = str(attempt.project_dir) - notice_started: float | None = _start_notice(attempt=attempt) - digests: dict[str, str] = with_missing_digests( - project_dir=project_dir, - snapshot=attempt.snapshot, - digests=attempt.digests, - paths=attempt.restamped, - snapshot_ns=attempt.snapshot_ns, + stored: bool = record_native( + store_path=attempt.entry_path, + native=attempt.native, + output=output, + exit_code=exit_code, + input_reads=input_reads, + artifact_writes=artifact_writes, + compile_cache_enabled=compile_cache_enabled, + artifacts_written=artifacts_written, + dag_artifact_path=dag_artifact_path, + json_output=json_output, ) - target_files: dict[str, FileStamp] | None = ( - verified_target_files( - project_dir=project_dir, - dag_path=None if dag_artifact_path is None else os.path.abspath(dag_artifact_path), - artifacts=artifact_writes.artifacts, - written=artifacts_written, - since_ns=attempt.snapshot_ns, - ) - if snapshot_project_files(project_dir=project_dir) == attempt.snapshot - else None - ) - if target_files is None: - remove_entry(path=entry_path) - _finish_notice(started=notice_started, message=REUSE_STORE_SKIPPED_MESSAGE) - return - environment_names: tuple[str, ...] = tracked_environment_names( - template_names=input_reads.environment_names - ) - write_entry( - path=entry_path, - inputs=StoredCompileInputs( - invocation_digest=attempt.invocation_digest, - runtime=attempt.runtime, - environment_names=environment_names, - environment_digest=environment_digest(names=environment_names), - search_path=attempt.search_path, - modules=loaded_module_stamps( - covered_paths=frozenset( - os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot - ), - extra_paths=macro_store_module_paths(), - ), - project_files=stored_project_files( - snapshot=attempt.snapshot, digests=digests, snapshot_ns=attempt.snapshot_ns - ), - target_files=target_files, - target_tree=artifacts_written, - settings_inputs=settings.inputs, - settings_digest=settings_inputs_digest(inputs=settings.inputs), - ), - output=StoredCompileOutput( - stderr_lines=output.stderr_lines, - exit_code=exit_code, - timings_span=compile_timings_span(stdout=stdout) if json_output else None, - stdout_length=len(stdout), - stdout_checksum=0, - stdout_file="", - ), - stdout=stdout, - ) - _finish_notice(started=notice_started, message=REUSE_STORE_DONE_MESSAGE) - - -def _start_notice(*, attempt: CompileReuseAttempt) -> float | None: - pending_bytes: int = pending_digest_bytes( - snapshot=attempt.snapshot, digests=attempt.digests, paths=attempt.restamped - ) - if pending_bytes < REUSE_STORE_NOTICE_BYTES and len(attempt.snapshot) < ( - REUSE_STORE_NOTICE_PATHS - ): - return None - print( - REUSE_STORE_START_MESSAGE.format( - paths=len(attempt.snapshot), mebibytes=pending_bytes / BYTES_PER_MEBIBYTE - ), - file=sys.stderr, - ) - return time.monotonic() - - -def _finish_notice(*, started: float | None, message: str) -> None: - if started is not None: - print(message.format(seconds=time.monotonic() - started), file=sys.stderr) + if stored: + write_launch_record(attempt=attempt) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py deleted file mode 100644 index 72d1fd4ecf..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Stat identity of the artifacts a compile leaves in target/.""" - -from __future__ import annotations - -import os - -from sqlbuild.cli.compile_reuse._helpers.project_files import file_digest -from sqlbuild.cli.compile_reuse.constants import ( - COMPILED_DIRECTORY_NAME, - FILE_KIND, - RACY_WINDOW_NS, - TARGET_DIRECTORY_NAME, -) -from sqlbuild.cli.compile_reuse.models import RecordedArtifact -from sqlbuild.cli.compile_reuse.types import FileStamp - - -def snapshot_target_files( - *, project_dir: str, extra_paths: tuple[str, ...] -) -> dict[str, FileStamp]: - """Stat every compiled artifact plus extra written artifacts, keyed by path.""" - - stamps: dict[str, FileStamp] = {} - pending: list[str] = [_compiled_root(project_dir=project_dir)] - while pending: - directory: str = pending.pop() - try: - entries: list[os.DirEntry[str]] = list(os.scandir(directory)) - except OSError: - continue - for entry in entries: - if entry.is_dir(follow_symlinks=False): - pending.append(entry.path) - continue - stamp: FileStamp | None = _path_stamp(path=entry.path) - if stamp is not None: - stamps[entry.path] = stamp - for path in extra_paths: - extra: FileStamp | None = _path_stamp(path=path) - if extra is not None: - stamps[path] = extra - return stamps - - -def verified_target_files( - *, - project_dir: str, - dag_path: str | None, - artifacts: dict[str, RecordedArtifact], - written: bool, - since_ns: int, -) -> dict[str, FileStamp] | None: - """Stamp this compile's artifacts, or return None when any changed since it wrote them.""" - - compiled_prefix: str = _compiled_root(project_dir=project_dir) + os.sep - expected: dict[str, RecordedArtifact] = { - path: artifact - for path, artifact in artifacts.items() - if path.startswith(compiled_prefix) or path == dag_path - } - stamps: dict[str, FileStamp] = _target_stamps( - project_dir=project_dir, paths=tuple(expected), tree=written - ) - if stamps.keys() != expected.keys() or not all( - _matches_recorded(path=path, stamp=stamp, artifact=expected[path], since_ns=since_ns) - for path, stamp in stamps.items() - ): - return None - if _target_stamps(project_dir=project_dir, paths=tuple(expected), tree=written) != stamps: - return None - return stamps - - -def target_files_unchanged(*, stored: dict[str, FileStamp], project_dir: str, tree: bool) -> bool: - """Return whether every stored artifact is unchanged and, for a written tree, none was added.""" - - return _target_stamps(project_dir=project_dir, paths=tuple(stored), tree=tree) == stored - - -def _target_stamps(*, project_dir: str, paths: tuple[str, ...], tree: bool) -> dict[str, FileStamp]: - """Stamp the whole compiled tree plus extra paths, or only the given paths.""" - - if not tree: - return {path: stamp for path in paths if (stamp := _path_stamp(path=path)) is not None} - compiled_prefix: str = _compiled_root(project_dir=project_dir) + os.sep - extra_paths: tuple[str, ...] = tuple( - path for path in paths if not path.startswith(compiled_prefix) - ) - return snapshot_target_files(project_dir=project_dir, extra_paths=extra_paths) - - -def _matches_recorded( - *, path: str, stamp: FileStamp, artifact: RecordedArtifact, since_ns: int -) -> bool: - if artifact.digest is None: - return stamp.size == artifact.size and stamp.mtime_ns == artifact.mtime_ns - if max(stamp.mtime_ns, stamp.ctime_ns) < since_ns - RACY_WINDOW_NS: - return True - return file_digest(path=path) == artifact.digest - - -def _compiled_root(*, project_dir: str) -> str: - return os.path.join(project_dir, TARGET_DIRECTORY_NAME, COMPILED_DIRECTORY_NAME) - - -def _path_stamp(*, path: str) -> FileStamp | None: - try: - status: os.stat_result = os.lstat(path) - except OSError: - return None - return FileStamp( - FILE_KIND, status.st_size, status.st_mtime_ns, status.st_ctime_ns, status.st_ino, None - ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/timings.py b/src/sqlbuild/cli/compile_reuse/_helpers/timings.py index 5ade4ea173..e33f7438b9 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/timings.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/timings.py @@ -1,14 +1,10 @@ -"""Locate and replace the compile_timings object inside a stored JSON report.""" +"""Compile reuse counters and elapsed time reported in compile_timings.""" from __future__ import annotations -import json import time from sqlbuild.cli.compile_reuse.constants import ( - COMPILE_TIMINGS_CLOSING, - COMPILE_TIMINGS_OPENING, - JSON_OBJECT_OPENING, MILLISECONDS_PER_SECOND, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, @@ -18,30 +14,6 @@ from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -def compile_timings_span(*, stdout: str) -> tuple[int, int] | None: - """Return the character span of the top-level compile_timings object, if any.""" - - opening: int = stdout.find(COMPILE_TIMINGS_OPENING) - if opening < 0: - return None - start: int = opening + len(COMPILE_TIMINGS_OPENING) - 1 - closing: int = stdout.find(COMPILE_TIMINGS_CLOSING, start) - if closing < 0 or JSON_OBJECT_OPENING in stdout[start + 1 : closing]: - return None - return start, closing + len(COMPILE_TIMINGS_CLOSING) - - -def replace_compile_timings(*, stdout: str, span: tuple[int, int], timings: dict[str, int]) -> str: - """Splice freshly measured timings into a stored two-space-indented JSON report.""" - - rendered: str = ( - "{\n" - + ",\n".join(f" {json.dumps(name)}: {value}" for name, value in timings.items()) - + COMPILE_TIMINGS_CLOSING - ) - return stdout[: span[0]] + rendered + stdout[span[1] :] - - def reuse_timings(*, outcome: CompileReuseOutcome, check_ms: int) -> dict[str, int]: """Return the reuse counters and check time reported in compile_timings.""" diff --git a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py index 069c0f24a9..4a895a8777 100644 --- a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py +++ b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py @@ -2,12 +2,12 @@ from __future__ import annotations -import hashlib import os import threading from collections.abc import Iterator from contextlib import contextmanager +import sqlbuild._native as _native from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites from sqlbuild.cli.compile_reuse.models import RecordedArtifact @@ -15,8 +15,7 @@ class CompileArtifactWriteRegistry: """Deliver artifact writes from every thread, including background staging threads.""" - def __init__(self, *, digest_size: int) -> None: - self._digest_size: int = digest_size + def __init__(self) -> None: self._lock: threading.Lock = threading.Lock() self._recorders: tuple[CompileArtifactWrites, ...] = () @@ -39,9 +38,7 @@ def written(self, *, path: str | os.PathLike[str], contents: bytes) -> None: recorders: tuple[CompileArtifactWrites, ...] = self._recorders if not recorders: return - artifact: RecordedArtifact = RecordedArtifact( - digest=hashlib.blake2b(contents, digest_size=self._digest_size).hexdigest() - ) + artifact: RecordedArtifact = RecordedArtifact(digest=_native.artifact_digest(contents)) for recorder in recorders: recorder.record(path=os.path.abspath(path), artifact=artifact) diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index d4da1c210d..2a6fc58080 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -2,8 +2,6 @@ from __future__ import annotations -from typing import Literal - from sqlbuild.cli.compile_reuse.classes.compile_artifact_write_registry import ( CompileArtifactWriteRegistry, ) @@ -15,15 +13,9 @@ from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR REUSE_FORMAT_VERSION: int = 1 -REUSE_ENTRY_DIRECTORY_NAME: str = "project-reuse-v1" -REUSE_ENTRY_MAGIC: bytes = b"SQBREUSE1\n" -REUSE_ENTRY_SUFFIX: str = ".entry" -REUSE_STDOUT_SUFFIX: str = ".stdout" -REUSE_RETIRED_RENDER_SUFFIX: str = ".render" -REUSE_STDOUT_SEPARATOR: str = "-" -REUSE_ENTRY_LENGTH_BYTES: int = 8 -REUSE_ENTRY_CHECKSUM_BYTES: int = 4 -REUSE_ENTRY_BYTE_ORDER: Literal["big"] = "big" +RETIRED_REUSE_DIRECTORY_NAME: str = "project-reuse-v1" +NATIVE_REUSE_DIRECTORY_NAME: str = "project-reuse-v2" +NATIVE_REUSE_SUFFIX: str = ".store" REUSE_MAX_ENTRY_BYTES: int = 512 * 1024 * 1024 REUSE_MAX_STORED_ENTRIES: int = 8 REUSE_DISABLE_ENV_VAR: str = COMPILE_REUSE_DISABLE_ENV_VAR @@ -43,8 +35,6 @@ PROJECT_CONFIG_FILENAMES: tuple[str, ...] = ("sqlbuild_project.toml", "sqlbuild_project.yml") RACY_WINDOW_NS: int = 2_000_000_000 -DIGEST_SIZE_BYTES: int = 16 -READ_CHUNK_BYTES: int = 1024 * 1024 MISSING_PATH_MTIME_NS: int = -1 PROJECT_ROOT_PATH_MTIME_NS: int = -2 OUTPUT_FILE_DESCRIPTORS: tuple[int, ...] = (1, 2) @@ -74,17 +64,6 @@ EXCLUDED_DIRECTORIES: frozenset[str] = frozenset({".git", "__pycache__"}) PRESENCE_ONLY_FILE_SUFFIXES: tuple[str, ...] = (".duckdb", ".duckdb.wal") -FILE_KIND: str = "f" -FILE_LINK_KIND: str = "fl" -DIRECTORY_KIND: str = "d" -DIRECTORY_LINK_KIND: str = "dl" -BROKEN_LINK_KIND: str = "bl" -SPECIAL_FILE_KIND: str = "s" -PRESENCE_ONLY_KIND: str = "p" - -COMPILE_TIMINGS_OPENING: str = '\n "compile_timings": {' -COMPILE_TIMINGS_CLOSING: str = "\n }" -JSON_OBJECT_OPENING: str = "{" REUSE_HIT_TIMING: str = "project_reuse_hits" REUSE_MISS_TIMING: str = "project_reuse_misses" REUSE_BYPASS_TIMING: str = "project_reuse_bypasses" @@ -102,6 +81,9 @@ REUSE_STORE_SKIPPED_MESSAGE: str = ( "Did not record compile for reuse: files changed while it ran ({seconds:.1f} s)" ) -COMPILE_ARTIFACT_WRITES: CompileArtifactWriteRegistry = CompileArtifactWriteRegistry( - digest_size=DIGEST_SIZE_BYTES -) +COMPILE_ARTIFACT_WRITES: CompileArtifactWriteRegistry = CompileArtifactWriteRegistry() +LAUNCH_RECORD_FORMAT: int = 1 +LAUNCH_RECORD_DIRECTORY: tuple[str, ...] = (TARGET_DIRECTORY_NAME, "cache", "launcher-v1") +LAUNCH_RECORD_LIMIT: int = 32 +COMPUTE_LOG_DIRECTORY_NAME: str = "logs" +COMPILE_COMMAND_NAME: str = "compile" diff --git a/src/sqlbuild/cli/compile_reuse/exceptions.py b/src/sqlbuild/cli/compile_reuse/exceptions.py deleted file mode 100644 index 7e360b538d..0000000000 --- a/src/sqlbuild/cli/compile_reuse/exceptions.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Compile reuse storage errors.""" - -from __future__ import annotations - - -class CompileReuseEntryError(ValueError): - """A stored compile is truncated, corrupt, or written in an unknown format.""" diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index cc2bd55c4b..0cf9624571 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -1,11 +1,12 @@ -"""Compile reuse requests, stored compile results, and per-invocation reuse state.""" +"""Compile reuse requests, provider settings inputs, and per-invocation reuse state.""" from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome, FileStamp +from sqlbuild._native import NativeReuseAttempt +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome @dataclass(frozen=True) @@ -57,60 +58,6 @@ class RecordedArtifact: mtime_ns: int | None = None -@dataclass(frozen=True) -class StoredProjectFile: - """One project path recorded before the stored compile ran.""" - - stamp: FileStamp - digest: str | None - racy: bool - - -@dataclass(frozen=True) -class StoredCompileInputs: - """Every compile input identity recorded with one stored compile result.""" - - invocation_digest: str - runtime: dict[str, str] - environment_names: tuple[str, ...] - environment_digest: str - search_path: tuple[tuple[str, int], ...] - modules: tuple[tuple[str, int, int], ...] - project_files: dict[str, StoredProjectFile] - target_files: dict[str, FileStamp] - target_tree: bool - settings_inputs: tuple[SettingsEnvironmentInputs, ...] - settings_digest: str - - -@dataclass(frozen=True) -class StoredCompileOutput: - """The command output of one stored compile, in emission order.""" - - stderr_lines: tuple[str, ...] - exit_code: int - timings_span: tuple[int, int] | None - stdout_length: int - stdout_checksum: int - stdout_file: str - - -@dataclass(frozen=True) -class StoredCompileHeader: - """Inputs and output description read without loading the stored stdout.""" - - inputs: StoredCompileInputs - output: StoredCompileOutput - - -@dataclass(frozen=True) -class ProjectFilesComparison: - """Whether project files are unchanged, with digests verified while comparing.""" - - unchanged: bool - verified: dict[str, str] - - @dataclass(frozen=True) class CompileReuseAttempt: """Result of checking for a reusable compile, carried into a full compile on a miss.""" @@ -121,10 +68,7 @@ class CompileReuseAttempt: check_ms: int = 0 exit_code: int | None = None entry_path: Path | None = None - invocation_digest: str = "" - runtime: dict[str, str] = field(default_factory=dict) - search_path: tuple[tuple[str, int], ...] = () - snapshot: dict[str, FileStamp] = field(default_factory=dict) - snapshot_ns: int = 0 - digests: dict[str, str] = field(default_factory=dict) - restamped: frozenset[str] = frozenset() + native: NativeReuseAttempt | None = field(default=None, compare=False, repr=False) + """The natively checked attempt, recorded natively after a miss.""" + selected_target: str | None = None + json_output: bool = False diff --git a/src/sqlbuild/cli/compile_reuse/types.py b/src/sqlbuild/cli/compile_reuse/types.py index e681ba99af..c592c4e642 100644 --- a/src/sqlbuild/cli/compile_reuse/types.py +++ b/src/sqlbuild/cli/compile_reuse/types.py @@ -1,9 +1,8 @@ -"""Compile reuse outcome and file stamp types.""" +"""Compile reuse outcome types.""" from __future__ import annotations from enum import StrEnum -from typing import NamedTuple class CompileReuseOutcome(StrEnum): @@ -12,14 +11,3 @@ class CompileReuseOutcome(StrEnum): HIT = "hit" MISS = "miss" BYPASS = "bypass" - - -class FileStamp(NamedTuple): - """Filesystem identity of one project path, as seen by a stat walk.""" - - kind: str - size: int - mtime_ns: int - ctime_ns: int - inode: int - link: str | None diff --git a/src/sqlbuild/cli/entry/_helpers/launch_state.py b/src/sqlbuild/cli/entry/_helpers/launch_state.py new file mode 100644 index 0000000000..d0bfd1330e --- /dev/null +++ b/src/sqlbuild/cli/entry/_helpers/launch_state.py @@ -0,0 +1,25 @@ +"""The launch record the native `sqb` executable offered this invocation.""" + +from contextvars import ContextVar, Token +from pathlib import Path + +_OFFERED: ContextVar[Path | None] = ContextVar("sqlbuild_launch_record", default=None) +_WITHHELD: ContextVar[bool] = ContextVar("sqlbuild_launch_record_withheld", default=False) + + +def offer_launch_record(path: Path) -> Token[Path | None]: + """Remember where `sqb` looks for this invocation's launch record.""" + + return _OFFERED.set(path) + + +def withhold_offered_launch_record() -> Token[bool]: + """Forget the offered launch record for the rest of this invocation.""" + + return _WITHHELD.set(True) + + +def offered_launch_record() -> Path | None: + """The offered launch record path, unless it was withheld.""" + + return None if _WITHHELD.get() else _OFFERED.get() diff --git a/src/sqlbuild/cli/entry/constants.py b/src/sqlbuild/cli/entry/constants.py index 3bff31b705..12e543d6ee 100644 --- a/src/sqlbuild/cli/entry/constants.py +++ b/src/sqlbuild/cli/entry/constants.py @@ -4,3 +4,4 @@ NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS: str = "10" OUTPUT_STREAM_ENCODING: str = "utf-8" OUTPUT_STREAM_ERRORS: str = "backslashreplace" +LAUNCH_RECORD_OPTION: str = "--sqlbuild-launch-record=" diff --git a/src/sqlbuild/cli/entry/main/_launch_record_path.py b/src/sqlbuild/cli/entry/main/_launch_record_path.py new file mode 100644 index 0000000000..9ba485c42a --- /dev/null +++ b/src/sqlbuild/cli/entry/main/_launch_record_path.py @@ -0,0 +1,11 @@ +"""Where this invocation may leave a launch record for the native `sqb` executable.""" + +from pathlib import Path + +from sqlbuild.cli.entry._helpers.launch_state import offered_launch_record + + +def launch_record_path() -> Path | None: + """The launch record path `sqb` offered, unless an input outside its key was read.""" + + return offered_launch_record() diff --git a/src/sqlbuild/cli/entry/main/_withhold_launch_record.py b/src/sqlbuild/cli/entry/main/_withhold_launch_record.py new file mode 100644 index 0000000000..fb5e4decf7 --- /dev/null +++ b/src/sqlbuild/cli/entry/main/_withhold_launch_record.py @@ -0,0 +1,9 @@ +"""Stop this invocation from leaving a launch record.""" + +from sqlbuild.cli.entry._helpers.launch_state import withhold_offered_launch_record + + +def withhold_launch_record() -> None: + """Withhold the launch record: `sqb` cannot replay what this invocation also depends on.""" + + _ = withhold_offered_launch_record() diff --git a/src/sqlbuild/cli/entry/main/launched.py b/src/sqlbuild/cli/entry/main/launched.py new file mode 100644 index 0000000000..60b51f56dc --- /dev/null +++ b/src/sqlbuild/cli/entry/main/launched.py @@ -0,0 +1,19 @@ +"""Python CLI entrypoint the native `sqb` executable hands every invocation it does not serve.""" + +import sys +from pathlib import Path + +from sqlbuild.cli.entry._helpers.launch_state import offer_launch_record +from sqlbuild.cli.entry.constants import LAUNCH_RECORD_OPTION +from sqlbuild.cli.entry.main.entry import main as run_cli + + +def main() -> int: + """Run the CLI with the arguments `sqb` received, remembering where its launch record goes.""" + + arguments: list[str] = sys.argv[1:] + if arguments and arguments[0].startswith(LAUNCH_RECORD_OPTION): + _ = offer_launch_record(Path(arguments[0].removeprefix(LAUNCH_RECORD_OPTION))) + arguments = arguments[1:] + sys.argv = [sys.argv[0], *arguments] + return run_cli() diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py index 4a93e94489..6699417d25 100644 --- a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py +++ b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py @@ -6,11 +6,10 @@ import sqlbuild._native as _native from sqlbuild.compiler.analysis_session._helpers.profile_rows import shape_rows -from sqlbuild.compiler.analysis_session._helpers.session_rows import contract_proof, family_rows +from sqlbuild.compiler.analysis_session._helpers.session_rows import family_rows from sqlbuild.compiler.analysis_session.models import NativePivotTables -from sqlbuild.compiler.analysis_session.types import ContractRow +from sqlbuild.compiler.analysis_session.types import ContractRow, ProofRow from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.compile.models import DynamicColumnContractProof from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily type _ModelRow = tuple[str, list[tuple[str, str, str, str, str, str | None]]] @@ -21,8 +20,8 @@ def native_dynamic_contracts( session: Any | None, tables: NativePivotTables, models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], -) -> tuple[DynamicColumnContractProof | None, ...]: - """Each model's proof in order; a native internal failure raises.""" +) -> tuple[ProofRow | None, ...]: + """Each model's proof row in order; a native internal failure raises.""" if not models: return () @@ -41,7 +40,7 @@ def native_dynamic_contracts( ) ) ) - return tuple(contract_proof(proof) for _, proof in contracts) + return tuple(proof for _, proof in contracts) def _model_row(*, sql: str, families: tuple[SchemaDynamicColumnFamily, ...]) -> _ModelRow: diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py b/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py index 61782def29..bcdc1804a5 100644 --- a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py +++ b/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py @@ -29,7 +29,7 @@ def native_model_analyses(*, request: NativeModelAnalysisRequest) -> NativeModel """Each model's analysis by name and the finished session; a native internal failure raises.""" if not request.model_inputs: - return NativeModelAnalyses(analyses={}, session=None) + return NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None) catalog: Any = cast(Any, request.inference_profile.binding_catalog) python: PythonModelAnalysis = PythonModelAnalysis( model_inputs=request.model_inputs, @@ -47,12 +47,7 @@ def native_model_analyses(*, request: NativeModelAnalysisRequest) -> NativeModel None if adapter_rules else _analysis_store(request), (adapter_rules, InferredNullability) if adapter_rules else None, ) - return NativeModelAnalyses( - analyses=NativeModelAnalysis(request=request, python=python, catalog=catalog).analyses( - session - ), - session=session, - ) + return NativeModelAnalysis(request=request, python=python, catalog=catalog).analyses(session) def _analysis_store(request: NativeModelAnalysisRequest) -> tuple[str, str] | None: diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py b/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py index 9da526a761..207aae147c 100644 --- a/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py +++ b/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py @@ -16,6 +16,7 @@ ) from sqlbuild.compiler.analysis_session.constants import ADAPTER_NULLABILITY_CALLBACK from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns +from sqlbuild.compiler.analysis_session.types import ColumnRow from sqlbuild.compiler.compile.models import CompileSqlReference, InferredColumn from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.lineage.types import InferredNullability @@ -44,8 +45,32 @@ def native_query_columns( ) -> tuple[NativeQueryColumns, ...]: """Each query's columns; a native internal failure raises `NativeCompilerError`.""" + return tuple( + NativeQueryColumns( + succeeded=succeeded, + columns=None if columns is None else _inferred_columns(columns), + has_star=has_star, + ) + for succeeded, columns, has_star in native_query_column_rows( + queries=queries, + profile=profile, + column_types_by_table=column_types_by_table, + column_nullability_by_table=column_nullability_by_table, + ) + ) + + +def native_query_column_rows( + *, + queries: Sequence[NativeColumnQuery], + profile: ExpressionInferenceProfile, + column_types_by_table: Mapping[str, Mapping[str, str]], + column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]], +) -> list[tuple[bool, list[ColumnRow] | None, bool]]: + """Each query's `(succeeded, column rows, has star)` as the native analysis returns it.""" + if not queries: - return () + return [] dialect: str = profile.sql_analysis_dialect or "generic" dialect = NATIVE_DIALECT_ALIASES.get(dialect, dialect) catalog: Any | None = ( @@ -65,28 +90,18 @@ def native_query_columns( adapter_rules: dict[str, FunctionNullabilityRule] = adapter_nullability_rules(profile) if adapter_rules: record_analysis_deferral(kind=ADAPTER_NULLABILITY_CALLBACK) - rows: list[tuple[bool, list[tuple[str, str | None, str]] | None, bool]] = ( - _native.infer_query_columns( - None if catalog is None else catalog.native, - ( - dialect, - list(profile.function_return_types.items()), - nullability_rule_rows(profile), - shape_rows(column_types_by_table), - shape_rows(column_nullability_by_table), - [_query_row(query) for query in queries], - profile.sql_analysis_dialect or "generic", - ), - (adapter_rules, InferredNullability) if adapter_rules else None, - ) - ) - return tuple( - NativeQueryColumns( - succeeded=succeeded, - columns=None if columns is None else _inferred_columns(columns), - has_star=has_star, - ) - for succeeded, columns, has_star in rows + return _native.infer_query_columns( + None if catalog is None else catalog.native, + ( + dialect, + list(profile.function_return_types.items()), + nullability_rule_rows(profile), + shape_rows(column_types_by_table), + shape_rows(column_nullability_by_table), + [_query_row(query) for query in queries], + profile.sql_analysis_dialect or "generic", + ), + (adapter_rules, InferredNullability) if adapter_rules else None, ) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py b/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py index 01ffc36212..c0566c871f 100644 --- a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py +++ b/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session._helpers.profile_rows import ( @@ -10,12 +10,14 @@ nullability_rule_rows, shape_rows, ) +from sqlbuild.compiler.analysis_session.constants import LINEAGE_NATIVE from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest from sqlbuild.compiler.analysis_session.types import ( ColumnRow, DiagnosticRow, FamilyRow, LineageItem, + OutcomeRow, ProofRow, ShapeRows, ) @@ -27,6 +29,8 @@ DynamicColumnContractProof, DynamicColumnFamilyProof, InferredColumn, + ModelSqlAnalysis, + PolyglotAnalysisResult, ) from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic @@ -60,6 +64,37 @@ def session_request( ) +def outcome_analysis( + *, outcome: OutcomeRow, contract: ProofRow | None, placeholders: dict[str, str] | None +) -> ModelSqlAnalysis: + """One finished session outcome as the `ModelSqlAnalysis` it replaced.""" + + succeeded, columns, source, rows, has_star, star_resolved, raw, validated, cleaned = outcome + return ModelSqlAnalysis( + polyglot_analysis=PolyglotAnalysisResult( + analysis_succeeded=succeeded, + columns=inferred_columns(columns), + lineage_columns=_lineage(succeeded=succeeded, source=source, rows=rows), + has_star=has_star, + star_resolved=star_resolved, + binding_diagnostics=binding_diagnostics(raw), + binding_validated=validated, + ), + placeholders=placeholders, + fused_binding_validated=True, + cleaned_sql=cleaned, + dynamic_column_contract=contract_proof(contract), + ) + + +def _lineage( + *, succeeded: bool, source: str, rows: list[LineageItem] +) -> Sequence[CompiledLineageColumnFact]: + if source != LINEAGE_NATIVE: + return lineage_facts(rows) + return compact_lineage(rows) if succeeded else () + + def contract_proof(row: ProofRow | None) -> DynamicColumnContractProof | None: """A native dynamic pivot proof as Python's, None when absent.""" @@ -79,27 +114,6 @@ def contract_proof(row: ProofRow | None) -> DynamicColumnContractProof | None: ) -def column_rows(columns: tuple[InferredColumn, ...] | None) -> list[ColumnRow] | None: - if columns is None: - return None - return [(column.name, column.type, str(column.nullability)) for column in columns] - - -def diagnostic_rows(diagnostics: tuple[SqlBindingDiagnostic, ...]) -> list[DiagnosticRow]: - return [ - ( - diagnostic.code, - diagnostic.message, - diagnostic.line, - diagnostic.column, - diagnostic.start, - diagnostic.end, - diagnostic.severity, - ) - for diagnostic in diagnostics - ] - - def inferred_columns(rows: list[ColumnRow] | None) -> tuple[InferredColumn, ...] | None: if rows is None: return None diff --git a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py b/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py index 9068596130..6fdec2125d 100644 --- a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py +++ b/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py @@ -1,40 +1,21 @@ -"""One compile's native model analysis, read into Python's analysis objects.""" +"""One compile's native model analysis session, with Python's catalog and metric bookkeeping.""" from __future__ import annotations import logging -from collections.abc import Sequence from typing import Any import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session._helpers.session_rows import ( - binding_diagnostics, - compact_lineage, - contract_proof, - lineage_facts, -) from sqlbuild.compiler.analysis_session.constants import ( - LINEAGE_NATIVE, NATIVE_ANALYSIS_FAILURE_MESSAGE, NATIVE_ANALYSIS_STORE_FAILURE_MESSAGE, ) -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest -from sqlbuild.compiler.analysis_session.types import ( - ColumnRow, - FinishRow, - OutcomeRow, - ProofRow, - ShapeRows, - StepRow, +from sqlbuild.compiler.analysis_session.models import ( + NativeModelAnalyses, + NativeModelAnalysisRequest, ) +from sqlbuild.compiler.analysis_session.types import FinishRow, ShapeRows, StepRow from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, - InferredColumn, - ModelSqlAnalysis, - ModelSqlAnalysisRequest, - PolyglotAnalysisResult, -) from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.diagnostics.main.log_debug_event import log_debug_event @@ -42,7 +23,7 @@ class NativeModelAnalysis: - """Drive one native session and read its outcomes as Python's analyses.""" + """Drive one native session; its outcomes stay native for compiled model assembly.""" def __init__( self, @@ -57,10 +38,9 @@ def __init__( self._nullability: dict[str, dict[str, InferredNullability]] = dict( request.column_nullability_by_table ) - self._columns: dict[ColumnRow, InferredColumn] = {} - def analyses(self, session: _native.NativeModelAnalysisSession) -> dict[str, ModelSqlAnalysis]: - """Every model's analysis by name; a native internal failure raises.""" + def analyses(self, session: _native.NativeModelAnalysisSession) -> NativeModelAnalyses: + """Every model's session position by name; a native internal failure raises.""" step: StepRow = session.run() publications, failures = step @@ -72,19 +52,21 @@ def analyses(self, session: _native.NativeModelAnalysisSession) -> dict[str, Mod ) self._publish(publications) finished: FinishRow = session.finish() - outcomes, schema_additions, analysis_names, contracts = finished + schema_additions, analysis_names, succeeded = finished self._record_catalog_changes( schema_additions=schema_additions, analysis_names=analysis_names ) self._record_cache(session) - return { - request.model_input.model_file.file_path.stem: self._model_analysis( - request=request, outcome=outcome, contract=contract - ) - for request, outcome, (_, contract) in zip( - self._python.requests, outcomes, contracts, strict=True - ) - } + names: tuple[str, ...] = tuple( + request.model_input.model_file.file_path.stem for request in self._python.requests + ) + return NativeModelAnalyses( + positions={name: position for position, name in enumerate(names)}, + succeeded=frozenset( + name for name, success in zip(names, succeeded, strict=True) if success + ), + session=session, + ) def _record_cache(self, session: _native.NativeModelAnalysisSession) -> None: stats: tuple[int, int, int, str | None] | None = session.cache_stats @@ -132,60 +114,3 @@ def _record_catalog_changes( if name in self._nullability }, ) - - def _inferred_columns(self, rows: list[ColumnRow] | None) -> tuple[InferredColumn, ...] | None: - """Columns as Python's analysis shares them: one object per distinct column value.""" - - if rows is None: - return None - return tuple(map(self._inferred_column, rows)) - - def _inferred_column(self, row: ColumnRow) -> InferredColumn: - column: InferredColumn | None = self._columns.get(row) - if column is None: - name, data_type, nullability = row - column = InferredColumn( - name=name, type=data_type, nullability=InferredNullability(nullability) - ) - self._columns[row] = column - return column - - def _model_analysis( - self, - *, - request: ModelSqlAnalysisRequest, - outcome: OutcomeRow, - contract: ProofRow | None, - ) -> ModelSqlAnalysis: - ( - succeeded, - columns, - lineage_source, - lineage_rows, - has_star, - star_resolved, - diagnostics, - validated, - cleaned_sql, - ) = outcome - lineage: Sequence[CompiledLineageColumnFact] = ( - (compact_lineage(lineage_rows) if succeeded else ()) - if lineage_source == LINEAGE_NATIVE - else lineage_facts(lineage_rows) - ) - return ModelSqlAnalysis( - polyglot_analysis=PolyglotAnalysisResult( - analysis_succeeded=succeeded, - columns=self._inferred_columns(columns), - lineage_columns=lineage, - has_star=has_star, - star_resolved=star_resolved, - binding_diagnostics=binding_diagnostics(diagnostics), - binding_validated=validated, - ), - placeholders=request.placeholders, - fused_binding_validated=True, - cleaned_sql=cleaned_sql, - validated_schema=request.binding_schema, - dynamic_column_contract=contract_proof(contract), - ) diff --git a/src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py b/src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py new file mode 100644 index 0000000000..2e058baa95 --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py @@ -0,0 +1,13 @@ +"""A native dynamic pivot proof row as Python's proof, for hand-built model inputs only.""" + +from __future__ import annotations + +from sqlbuild.compiler.analysis_session._helpers.session_rows import contract_proof +from sqlbuild.compiler.analysis_session.types import ProofRow +from sqlbuild.compiler.compile.models import DynamicColumnContractProof + + +def dynamic_contract_proof(row: ProofRow | None) -> DynamicColumnContractProof | None: + """The proof, or None when the row is absent.""" + + return contract_proof(row) diff --git a/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py b/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py new file mode 100644 index 0000000000..2d44728cbf --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py @@ -0,0 +1,27 @@ +"""Infer query output column rows natively outside the analysis session.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.analysis_session._helpers.query_columns import native_query_column_rows +from sqlbuild.compiler.analysis_session.models import NativeColumnQuery +from sqlbuild.compiler.analysis_session.types import ColumnRow +from sqlbuild.compiler.lineage.types import InferredNullability + + +def infer_native_query_column_rows( + *, + queries: Sequence[NativeColumnQuery], + profile: ExpressionInferenceProfile, + column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]] | None = None, +) -> list[tuple[bool, list[ColumnRow] | None, bool]]: + """Each query's `(succeeded, column rows, has star)`; a native internal failure raises.""" + + return native_query_column_rows( + queries=queries, + profile=profile, + column_types_by_table={}, + column_nullability_by_table=column_nullability_by_table or {}, + ) diff --git a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py index 4f01fb1e9f..3565ea75a6 100644 --- a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py +++ b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py @@ -6,7 +6,7 @@ from sqlbuild.compiler.analysis_session._helpers.dynamic_contracts import native_dynamic_contracts from sqlbuild.compiler.analysis_session.models import NativePivotTables -from sqlbuild.compiler.compile.models import DynamicColumnContractProof +from sqlbuild.compiler.analysis_session.types import ProofRow from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -15,7 +15,7 @@ def prove_native_dynamic_contracts( session: Any | None, tables: NativePivotTables, models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], -) -> tuple[DynamicColumnContractProof | None, ...]: - """Python's proof per `(sql, families)` model from the session or `tables`; None for Python.""" +) -> tuple[ProofRow | None, ...]: + """The proof row per `(sql, families)` model from the session or `tables`, else None.""" return native_dynamic_contracts(session=session, tables=tables, models=models) diff --git a/src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py b/src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py new file mode 100644 index 0000000000..74342b9b6a --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py @@ -0,0 +1,20 @@ +"""A finished session outcome as Python's analysis, for hand-built model inputs only.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.compiler.analysis_session._helpers.session_rows import outcome_analysis +from sqlbuild.compiler.analysis_session.types import ContractRow, OutcomeRow +from sqlbuild.compiler.compile.models import ModelSqlAnalysis + + +def session_outcome_analysis( + *, session: Any, position: int, placeholders: dict[str, str] | None +) -> ModelSqlAnalysis: + """The outcome at `position` of a finished session as an explicit boundary conversion.""" + + outcome: OutcomeRow + contract: ContractRow + outcome, contract = session.outcome_rows(position) + return outcome_analysis(outcome=outcome, contract=contract[1], placeholders=placeholders) diff --git a/src/sqlbuild/compiler/analysis_session/models.py b/src/sqlbuild/compiler/analysis_session/models.py index 3fd27e09bc..11c5082f04 100644 --- a/src/sqlbuild/compiler/analysis_session/models.py +++ b/src/sqlbuild/compiler/analysis_session/models.py @@ -9,11 +9,10 @@ from sqlbuild.compiler.analysis_session.types import QueryColumnsMode from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, - CompileModelInput, CompileSqlReference, InferredColumn, - ModelSqlAnalysis, ) +from sqlbuild.compiler.compile.types import ModelInput from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -24,7 +23,7 @@ class NativeModelAnalysisRequest: known_functions: tuple[str, ...] known_types: tuple[str, ...] - model_inputs: tuple[CompileModelInput, ...] + model_inputs: tuple[ModelInput, ...] column_nullability_by_table: dict[str, dict[str, InferredNullability]] column_types_by_table: dict[str, dict[str, str]] inference_profile: ExpressionInferenceProfile @@ -37,9 +36,10 @@ class NativeModelAnalysisRequest: @dataclass(frozen=True, slots=True) class NativeModelAnalyses: - """Each model's native analysis by name, and the finished session that produced them.""" + """Each analysed model's position in the finished session that keeps its outcome.""" - analyses: dict[str, ModelSqlAnalysis] + positions: dict[str, int] + succeeded: frozenset[str] session: Any | None diff --git a/src/sqlbuild/compiler/analysis_session/types.py b/src/sqlbuild/compiler/analysis_session/types.py index bfa4f7e783..41e9822db7 100644 --- a/src/sqlbuild/compiler/analysis_session/types.py +++ b/src/sqlbuild/compiler/analysis_session/types.py @@ -25,6 +25,6 @@ bool, list[ColumnRow], list[tuple[str, str | None]], list[str], str | None, bool ] type ContractRow = tuple[str, ProofRow | None] -type FinishRow = tuple[list[OutcomeRow], ShapeRows, list[str], list[ContractRow]] +type FinishRow = tuple[ShapeRows, list[str], list[bool]] type QueryColumnsMode = Literal["batch", "reanalysis", "parse", "legacy"] diff --git a/src/sqlbuild/compiler/attachments/__init__.py b/src/sqlbuild/compiler/attachments/__init__.py deleted file mode 100644 index cc22a420ba..0000000000 --- a/src/sqlbuild/compiler/attachments/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native compile attachments for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/attachments/constants.py b/src/sqlbuild/compiler/attachments/constants.py deleted file mode 100644 index 25136884df..0000000000 --- a/src/sqlbuild/compiler/attachments/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Values the native compile attachments return.""" - -INSTANCE_POLICY: str = "instance" -DEFAULT_POLICY: str = "default" -HEADER_START_STAGE: str = "start" -HEADER_ARGUMENTS_STAGE: str = "arguments" -HEADER_RETURNS_STAGE: str = "returns" -HEADER_PYTHON_VALUES_STAGE: str = "python_values" -HEADER_METADATA_STAGE: str = "metadata" diff --git a/src/sqlbuild/compiler/attachments/main/__init__.py b/src/sqlbuild/compiler/attachments/main/__init__.py deleted file mode 100644 index 35e81c6936..0000000000 --- a/src/sqlbuild/compiler/attachments/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native compile attachment entry points.""" diff --git a/src/sqlbuild/compiler/attachments/main/_native_test_target_catalog.py b/src/sqlbuild/compiler/attachments/main/_native_test_target_catalog.py deleted file mode 100644 index ce59be2df8..0000000000 --- a/src/sqlbuild/compiler/attachments/main/_native_test_target_catalog.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Build the native SQL test target catalog once per compile for the preview compiler engine.""" - -from __future__ import annotations - -from collections.abc import Collection - -import sqlbuild._native as _native - - -def native_test_target_catalog( - *, - models: Collection[str] = (), - sources: Collection[str] = (), - seeds: Collection[str] = (), - table_functions: Collection[str] = (), - macros: Collection[str] = (), -) -> _native.SqlTestTargetCatalog: - """Return the project's resource names as one native catalog shared by every test.""" - - return _native.SqlTestTargetCatalog( - set(models), set(sources), set(seeds), (set(table_functions), set(macros)) - ) diff --git a/src/sqlbuild/compiler/attachments/main/_pair_native_seed_files.py b/src/sqlbuild/compiler/attachments/main/_pair_native_seed_files.py deleted file mode 100644 index 822fdac33b..0000000000 --- a/src/sqlbuild/compiler/attachments/main/_pair_native_seed_files.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Pair seed declarations with seed CSV files natively for the preview compiler engine.""" - -from __future__ import annotations - -import sqlbuild._native as _native - - -def pair_native_seed_files( - *, declaration_names: list[str], file_stems: list[str] -) -> tuple[list[int], None] | tuple[None, int]: - """Return each declaration's seed file index, or the index of the first one without a file.""" - - return _native.pair_seed_files(declaration_names, file_stems) diff --git a/src/sqlbuild/compiler/attachments/main/_parse_native_function_header.py b/src/sqlbuild/compiler/attachments/main/_parse_native_function_header.py deleted file mode 100644 index 387d0630e7..0000000000 --- a/src/sqlbuild/compiler/attachments/main/_parse_native_function_header.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Parse a SQL or Python function header natively.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.attachments.models import NativeFunctionHeader, NativeNamedType - - -def parse_native_function_header( - *, header_values: dict[str, object], python: bool, relative_path: Path -) -> NativeFunctionHeader: - """Return the parsed header with its first error.""" - - row: tuple[ - list[tuple[str, str, str]], - str | None, - list[tuple[str, str, str]] | None, - list[str], - str | None, - str | None, - str | None, - list[str], - tuple[str, str] | None, - ] = _native.parse_function_header_values(header_values, python, str(relative_path)) - ( - arguments, - returns, - columns, - tags, - description, - runtime_version, - entry_point, - packages, - failure, - ) = row - return NativeFunctionHeader( - arguments=_named_types(arguments), - returns=returns, - return_columns=None if columns is None else _named_types(columns), - tags=tuple(tags), - description=description, - runtime_version=runtime_version, - entry_point=entry_point, - packages=tuple(packages), - failure=failure, - ) - - -def _named_types(rows: list[tuple[str, str, str]]) -> tuple[NativeNamedType, ...]: - return tuple(NativeNamedType(*row) for row in rows) diff --git a/src/sqlbuild/compiler/attachments/main/_render_native_attached_audit.py b/src/sqlbuild/compiler/attachments/main/_render_native_attached_audit.py deleted file mode 100644 index 627a80dd5d..0000000000 --- a/src/sqlbuild/compiler/attachments/main/_render_native_attached_audit.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Render one attached generic audit natively.""" - -from __future__ import annotations - -from dataclasses import asdict - -import sqlbuild._native as _native -from sqlbuild.compiler.attachments.models import NativeAuditPolicies, NativeRenderedAudit - - -def render_native_attached_audit( - *, - labels: tuple[str, str], - sql_body: str, - evidence_sql: str | None, - implicit_arguments: dict[str, str], - explicit_arguments: dict[str, object], - policies: NativeAuditPolicies, -) -> NativeRenderedAudit: - """Return the rendering of `(owner, audit)`, or its render and policy errors.""" - - return NativeRenderedAudit( - *_native.render_attached_generic_audit( - labels, - (sql_body, evidence_sql), - (list(implicit_arguments.items()), explicit_arguments), - asdict(policies), - ) - ) diff --git a/src/sqlbuild/compiler/attachments/main/_resolve_native_function_namespace.py b/src/sqlbuild/compiler/attachments/main/_resolve_native_function_namespace.py deleted file mode 100644 index a30bfbcac1..0000000000 --- a/src/sqlbuild/compiler/attachments/main/_resolve_native_function_namespace.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Resolve a function's namespace natively for the preview compiler engine.""" - -from __future__ import annotations - -from dataclasses import asdict - -import sqlbuild._native as _native -from sqlbuild.compiler.attachments.models import ( - NativeFunctionNamespace, - NativeFunctionNamespaceInputs, -) - - -def resolve_native_function_namespace( - inputs: NativeFunctionNamespaceInputs, -) -> NativeFunctionNamespace: - """Return the physical, logical and fingerprint namespace Python's attachment assigns.""" - - return NativeFunctionNamespace(*_native.resolve_function_namespace_values(asdict(inputs))) diff --git a/src/sqlbuild/compiler/attachments/models.py b/src/sqlbuild/compiler/attachments/models.py deleted file mode 100644 index f070f514af..0000000000 --- a/src/sqlbuild/compiler/attachments/models.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Plain values exchanged with the native compile attachments.""" - -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True, slots=True) -class NativeAuditPolicies: - """One attachment's threshold facts, severities and run scopes.""" - - has_thresholds: bool - threshold_error: bool - instance_severity: str | None - default_severity: str | None - instance_run_scope: str | None - default_run_scope: str | None - - -@dataclass(frozen=True, slots=True) -class NativeRenderedAudit: - """Rendered audit SQL and policies, or the render error and the later policy error.""" - - render_error: str | None - sql_body: str - evidence_sql: str | None - severity: str - run_scope_source: str - policy_error: str | None - - -@dataclass(frozen=True, slots=True) -class NativeNamedType: - """One function argument or return column: authored name, stripped name, stripped type.""" - - raw_name: str - name: str - type_text: str - - -@dataclass(frozen=True, slots=True) -class NativeFunctionHeader: - """A function header with unexpanded types and Python's first `(stage, message)` error.""" - - arguments: tuple[NativeNamedType, ...] - returns: str | None - return_columns: tuple[NativeNamedType, ...] | None - tags: tuple[str, ...] - description: str | None - runtime_version: str | None - entry_point: str | None - packages: tuple[str, ...] - failure: tuple[str, str] | None - - -@dataclass(frozen=True, slots=True) -class NativeFunctionNamespaceInputs: - """Expanded header, project default and target namespaces of one function.""" - - header_database: str | None - header_schema: str | None - default_database: str | None - default_schema: str | None - target_database: str | None - target_schema: str | None - python: bool - inherit_default_namespace: bool - - -@dataclass(frozen=True, slots=True) -class NativeFunctionNamespace: - """Physical, logical and fingerprint database and schema of one function.""" - - database: str | None - schema: str | None - logical_database: str | None - logical_schema: str | None - fingerprint_database: str | None - fingerprint_schema: str | None - fingerprint_logical_database: str | None - fingerprint_logical_schema: str | None diff --git a/src/sqlbuild/compiler/auditing/constants.py b/src/sqlbuild/compiler/auditing/constants.py index 265b41902f..55ad22b4db 100644 --- a/src/sqlbuild/compiler/auditing/constants.py +++ b/src/sqlbuild/compiler/auditing/constants.py @@ -6,8 +6,6 @@ BUILT_IN_AUDIT_SHADOW_CODE: str = "P003" MEASUREMENT_THRESHOLD_WARN_KEY: str = "warn" MEASUREMENT_THRESHOLD_ERROR_KEY: str = "error" -MEASUREMENT_THRESHOLDS_HEADER_KEY: str = "thresholds" -MEASUREMENT_MINIMUM_SAMPLES_HEADER_KEY: str = "minimum_samples" MEASUREMENT_OUTSIDE_BOUND_VALUE_COUNT: int = 2 SCHEMA_AUDIT_OPTION_KEYS: frozenset[str] = frozenset( { diff --git a/src/sqlbuild/compiler/auditing/main/_builtins.py b/src/sqlbuild/compiler/auditing/main/_builtins.py index 2499d19c3f..db5409dd37 100644 --- a/src/sqlbuild/compiler/auditing/main/_builtins.py +++ b/src/sqlbuild/compiler/auditing/main/_builtins.py @@ -5,24 +5,13 @@ from pathlib import Path from textwrap import dedent -from sqlbuild.compiler.auditing.constants import ( - BUILT_IN_AUDIT_NAMES, - BUILT_IN_AUDIT_SHADOW_CODE, -) -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity from sqlbuild.compiler.discovery.models import DiscoveredAuditBlock, DiscoveredAuditFile -def build_builtin_audit_resolution( - project_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]], -) -> tuple[ - dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]], - tuple[CompilerDiagnostic, ...], -]: - """Return built-in-aware generic audit definitions and shadow diagnostics.""" +def builtin_audit_definitions() -> tuple[DiscoveredAuditFile, ...]: + """Every built-in generic audit definition, which project definitions override by name.""" - merged: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] = { + definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] = { "accepted_values": _build_audit_definition( name="accepted_values", sql_body=dedent( @@ -72,35 +61,7 @@ def build_builtin_audit_resolution( ).strip(), ), } - merged.update(project_definitions) - return merged, _build_builtin_audit_shadow_diagnostics(project_definitions) - - -def _build_builtin_audit_shadow_diagnostics( - project_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]], -) -> tuple[CompilerDiagnostic, ...]: - """Build warnings for project generic audits that override built-ins.""" - - diagnostics: list[CompilerDiagnostic] = [] - definition_name: str - definition: tuple[DiscoveredAuditFile, DiscoveredAuditBlock] - for definition_name, definition in sorted(project_definitions.items()): - if definition_name not in BUILT_IN_AUDIT_NAMES: - continue - diagnostics.append( - CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.WARNING, - code=BUILT_IN_AUDIT_SHADOW_CODE, - message=( - f"project audit '{definition_name}' overrides built-in audit " - f"'{definition_name}'" - ), - path=definition[0].relative_path, - help="rename the project audit if this override is unintended", - ) - ) - return tuple(diagnostics) + return tuple(audit_file for audit_file, _ in definitions.values()) def _build_audit_definition( diff --git a/src/sqlbuild/compiler/auditing/main/_parse_audit_instance.py b/src/sqlbuild/compiler/auditing/main/_parse_audit_instance.py index 5a210cfbb1..89ff3352b6 100644 --- a/src/sqlbuild/compiler/auditing/main/_parse_audit_instance.py +++ b/src/sqlbuild/compiler/auditing/main/_parse_audit_instance.py @@ -4,8 +4,6 @@ from sqlbuild.compiler.auditing._helpers.schema_audits import ( parse_audit_instance_impl, - parse_measurement_thresholds, # noqa: F401 - sibling-domain role export - parse_minimum_samples, # noqa: F401 - sibling-domain role export ) from sqlbuild.spec.contracts.models import SchemaAuditInstance diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py b/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py index 33861ab3aa..fc87926863 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/pivot_requests.py @@ -11,7 +11,7 @@ from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( cursor_intrinsics_analysis_sql, ) -from sqlbuild.compiler.compile.models import CompileModelInput, ModelSqlAnalysis +from sqlbuild.compiler.compile.types import ModelInput from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -23,7 +23,7 @@ def model_pivot_sql(*, query_sql: str, placeholders: dict[str, str] | None) -> s return substitute_placeholder_defaults(query_sql=query_sql, placeholders=placeholders) -def model_dynamic_families(model_input: CompileModelInput) -> tuple[SchemaDynamicColumnFamily, ...]: +def model_dynamic_families(model_input: ModelInput) -> tuple[SchemaDynamicColumnFamily, ...]: """Return the dynamic column families the model's schema declares.""" if model_input.schema_entry is None: @@ -32,14 +32,14 @@ def model_dynamic_families(model_input: CompileModelInput) -> tuple[SchemaDynami def standalone_pivot_models( - *, model_inputs: tuple[CompileModelInput, ...], analyses: Mapping[str, ModelSqlAnalysis] + *, model_inputs: tuple[ModelInput, ...], analysed: Mapping[str, int] ) -> dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]]: """Return `(sql, families)` by model index for each pivot model no analysis covered.""" models: dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]] = {} for index, model_input in enumerate(model_inputs): families: tuple[SchemaDynamicColumnFamily, ...] = model_dynamic_families(model_input) - if not families or model_input.model_file.file_path.stem in analyses: + if not families or model_input.model_file.file_path.stem in analysed: continue query_sql: str = cursor_intrinsics_analysis_sql( sql=model_input.query_sql, diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py b/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py index 7e58ca1562..e700fb00fa 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/syntax_checks.py @@ -2,23 +2,16 @@ from __future__ import annotations -import re - from sqlbuild.compiler.compile._helpers.analysis.validation import hook_sql_statements from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( cursor_intrinsics_analysis_sql, ) -from sqlbuild.compiler.compile.models import CompileModelInput +from sqlbuild.compiler.compile.types import ModelInput _HOOK_NAMES: tuple[str, ...] = ("pre_hooks", "post_hooks") -_SET_OPERATION_PATTERN: re.Pattern[str] = re.compile( - r"\b(?:UNION|INTERSECT|EXCEPT)\b", re.IGNORECASE -) -_SET_OPERATION_KEYWORDS: tuple[str, ...] = ("UNION", "INTERSECT", "EXCEPT") -_DOTTED_CAPITAL_I: str = "\u0130" -def model_placeholders(model_input: CompileModelInput) -> dict[str, str] | None: +def model_placeholders(model_input: ModelInput) -> dict[str, str] | None: """Return the model's `@@@name` placeholder defaults as strings, or None.""" raw_placeholders: object | None = model_input.config.values.get("placeholders") @@ -31,7 +24,7 @@ def model_placeholders(model_input: CompileModelInput) -> dict[str, str] | None: def model_syntax_checks( *, - model_inputs: tuple[CompileModelInput, ...], + model_inputs: tuple[ModelInput, ...], analysis_model_names: frozenset[str] | None, analysis_succeeded: frozenset[str], ) -> tuple[tuple[tuple[str, dict[str, str] | None], ...], ...]: @@ -49,7 +42,7 @@ def model_syntax_checks( def _syntax_checks( *, - model_input: CompileModelInput, + model_input: ModelInput, analysis_model_names: frozenset[str] | None, analysis_succeeded: frozenset[str], ) -> tuple[tuple[str, dict[str, str] | None], ...]: @@ -70,20 +63,3 @@ def _syntax_checks( ) ) return tuple((statement, placeholders) for statement in statements) - - -def names_set_operation(sql: str) -> bool: - """Whether the case-insensitive search for `UNION`, `INTERSECT` or `EXCEPT` matches.""" - - folded: str = sql.upper().replace(_DOTTED_CAPITAL_I, "I") - if len(folded) != len(sql): - return any(keyword in folded for keyword in _SET_OPERATION_KEYWORDS) and ( - _SET_OPERATION_PATTERN.search(sql) is not None - ) - for keyword in _SET_OPERATION_KEYWORDS: - offset: int = folded.find(keyword) - while offset >= 0: - if _SET_OPERATION_PATTERN.match(sql, offset) is not None: - return True - offset = folded.find(keyword, offset + 1) - return False diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py index 80b6228992..a256b85574 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/validation.py @@ -37,22 +37,6 @@ def validate_sql_syntax( ) from None -def validate_function_sql_syntax( - *, - body_sql: str, - function_name: str, - file_path: Path, - placeholders: dict[str, str] | None = None, -) -> None: - """Validate that a SQL function body is parseable by Polyglot.""" - - _validate_sql_syntax_with_message( - query_sql=body_sql, - error_prefix=f"SQL syntax error in function '{function_name}' ({file_path})", - placeholders=placeholders, - ) - - def validate_hook_sql_syntax( *, value: object, @@ -132,44 +116,6 @@ def hook_sql_statements(value: object) -> tuple[str, ...]: return tuple(statements) -def validate_source_expression_syntax( - *, - expression: str, - source_name: str, - file_path: Path, -) -> None: - """Validate that a source expression is parseable as a FROM target.""" - - from sqlbuild.compiler.references.main.render_source_relation import render_source_relation - from sqlbuild.spec.contracts.models import SourceEntry - - rendered: str = render_source_relation( - entry=SourceEntry(name=source_name, expression=expression) - ) - _validate_sql_syntax_with_message( - query_sql=f"SELECT * FROM {rendered}", - error_prefix=f"SQL syntax error in source expression '{source_name}' ({file_path})", - ) - - -def _validate_sql_syntax_with_message( - *, - query_sql: str, - error_prefix: str, - placeholders: dict[str, str] | None = None, - dialect: str | None = None, -) -> None: - """Parse one SQL expression with Polyglot and raise a contextual error.""" - - error_message: str | None = _native.sql_syntax_error( - query_sql, placeholders, dialect, parse_one=True - ) - if error_message is not None: - _raise_sql_validation_error( - error_prefix=error_prefix, error_message=error_message, model_owned=False - ) - - def _validate_hook_sql_with_message( *, query_sql: str, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py index 4e265b80ea..1388e76984 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py @@ -5,20 +5,12 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import CompiledSqlExpansion, CompileModelInput, ExpansionSpan +from sqlbuild.compiler.compile.models import CompiledSqlExpansion, ExpansionSpan from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic from sqlbuild.compiler.sql_analysis.types import NativeBindingPositions, NativePositionsModule from sqlbuild.spec.contracts.models import SourceLocation -def query_line_offset(model_input: CompileModelInput) -> int | None: - query_sql: str = model_input.model_file.query_sql - query_start: int = model_input.model_file.contents.find(query_sql) - if query_start < 0: - return None - return model_input.model_file.contents[:query_start].count("\n") - - @lru_cache(maxsize=32) def _mapper( *, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py b/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py new file mode 100644 index 0000000000..0bfa744cc6 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py @@ -0,0 +1,164 @@ +"""Explicit boundary conversion of a hand-built model input's native analysis.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import dynamic_contract_proof +from sqlbuild.compiler.analysis_session.main._session_outcome_analysis import ( + session_outcome_analysis, +) +from sqlbuild.compiler.compile._helpers.assembly.targets import build_model_relation_target +from sqlbuild.compiler.compile.models import ( + CompiledLineageColumnFact, + CompiledModel, + CompiledObjectKey, + CompiledSqlExpansion, + CompilerDiagnostic, + DynamicColumnContractProof, + ExpansionSpan, + HandBuiltAnalysis, + InferredColumn, + ModelSqlAnalysis, + PolyglotAnalysisResult, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + ModelInput, +) +from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from sqlbuild.spec.contracts.models import SourceLocation + + +def hand_built_compiled_model( + *, + model_input: ModelInput, + deps: list[tuple[str, str]], + analysis: HandBuiltAnalysis, + macro_deps: tuple[str, ...], +) -> CompiledModel: + """A `CompiledModel` for a model input built outside the native compile-input stage.""" + + model_name: str = model_input.model_file.file_path.stem + inferred: tuple[InferredColumn, ...] | None = None + lineage: Sequence[CompiledLineageColumnFact] | None = None + has_star: bool = False + star_resolved: bool = False + diagnostics: tuple[CompilerDiagnostic, ...] = () + validated: bool = False + proof: DynamicColumnContractProof | None = dynamic_contract_proof(analysis.standalone_proof) + if analysis.session is not None: + session, position = analysis.session + outcome: ModelSqlAnalysis = session_outcome_analysis( + session=session, position=position, placeholders=None + ) + result: PolyglotAnalysisResult = outcome.polyglot_analysis + if result.analysis_succeeded: + inferred = result.columns + lineage = result.lineage_columns + has_star, star_resolved = result.has_star, result.star_resolved + else: + inferred = dynamic_fallback_columns(analysis.fallback) + proof = outcome.dynamic_column_contract or proof + validated = result.binding_validated + diagnostics = _located( + model_input=model_input, + diagnostics=result.binding_diagnostics, + cleaned=outcome.cleaned_sql or "", + ) + if proof is not None and proof.output_proven: + inferred = proof.fixed_columns + has_star = True + return CompiledModel( + key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=model_name), + deps=tuple( + CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name) + for kind, name in deps + ), + name=model_name, + relative_path=model_input.model_file.relative_path, + query_sql=model_input.query_sql, + config=model_input.config, + destination=build_model_relation_target(model_input=model_input, model_name=model_name), + references=model_input.references, + schema_entry=model_input.schema_entry, + inferred_columns=inferred, + fast_lineage_columns=lineage, + fast_lineage_has_star=has_star, + fast_lineage_star_resolved=star_resolved, + authored_sql=model_input.model_file.contents, + authored_query_sql=model_input.model_file.query_sql, + output_column_locations=model_input.model_file.output_column_locations, + extract_implicit_alias_columns=model_input.model_file.extract_implicit_alias_columns, + macro_deps=macro_deps, + enum_declarations=model_input.enum_declarations, + constant_declarations=model_input.constant_declarations, + enum_columns=model_input.enum_columns, + binding_diagnostics=diagnostics, + binding_validated=validated, + dynamic_column_contract=proof, + rejected_sql_analysis_opt_out=model_input.rejected_sql_analysis_opt_out, + ) + + +def dynamic_fallback_columns( + rows: list[tuple[str, str | None, str]] | None, +) -> tuple[InferredColumn, ...] | None: + """A failed analysis's parse-mode column rows as inferred columns.""" + + if rows is None: + return None + return tuple( + InferredColumn(name=name, type=data_type, nullability=InferredNullability(nullability)) + for name, data_type, nullability in rows + ) + + +def _located( + *, model_input: ModelInput, diagnostics: tuple[SqlBindingDiagnostic, ...], cleaned: str +) -> tuple[CompilerDiagnostic, ...]: + if not diagnostics: + return () + expansion: CompiledSqlExpansion | None = model_input.sql_expansion + path: Path = model_input.model_file.relative_path + located: list[ + tuple[str, str, str, int | None, int | None, tuple[int, int, int | None, int | None] | None] + ] = _native.locate_model_binding_diagnostics( + [ + (item.code, item.message, item.line, item.column, item.start, item.end, item.severity) + for item in diagnostics + ], + (model_input.model_file.contents, model_input.model_file.query_sql, cleaned), + None if expansion is None else (expansion.expanded_sql, _pass_rows(expansion)), + ) + return tuple( + CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=severity, + code=code, + message=message, + resource_type=CompiledResourceType.MODEL, + resource_name=model_input.model_file.file_path.stem, + path=path, + line=line, + column=column, + location=None + if span is None + else SourceLocation( + path=path, line=span[0], column=span[1], end_line=span[2], end_column=span[3] + ), + ) + for code, message, severity, line, column, span in located + ) + + +def _pass_rows(expansion: CompiledSqlExpansion) -> list[list[tuple[int, int, int, int]]]: + return [list(map(_span_row, spans)) for spans in expansion.passes] + + +def _span_row(span: ExpansionSpan) -> tuple[int, int, int, int]: + return (span.source_start, span.source_end, span.output_start, span.output_end) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py index 6a3fdff5fd..6c496aef8d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py @@ -1,192 +1,18 @@ -"""Check column-bearing model metadata against authoritative output shapes.""" +"""Compile-owned inputs of the native semantic metadata checks.""" from __future__ import annotations -import re -from functools import lru_cache - from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( infer_native_query_columns, ) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( - known_declared_types, - known_function_names, -) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledModelSqlTestPayload, - CompiledProject, - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.sql_analysis.constants import BINDING_UNKNOWN_TABLE_INTERNAL_CODE -from sqlbuild.compiler.sql_analysis.main._resolve_binding_column import resolve_binding_column -from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations -from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest -from sqlbuild.spec.contracts.models import SchemaAuditInstance - -_EXPRESSION_AUDIT: str = "expression_is_true" -_RELATIONSHIPS_AUDIT: str = "relationships" -_ACCEPTED_VALUES_AUDIT: str = "accepted_values" - - -def _audit_errors( - *, - model: CompiledModel, - project: CompiledProject, - shape: dict[str, str], - shapes: dict[str, dict[str, str]], - profile: ExpressionInferenceProfile, -) -> tuple[CompilerDiagnostic, ...]: - if model.schema_entry is None: - return () - diagnostics: list[CompilerDiagnostic] = [] - attached: list[tuple[str | None, SchemaAuditInstance]] = [ - (None, audit) for audit in model.schema_entry.audits - ] - for schema_column in model.schema_entry.columns: - attached.extend((schema_column.name, audit) for audit in schema_column.audits) - for column, audit in attached: - arguments: dict[str, object] = audit.arguments - if audit.definition_name == _EXPRESSION_AUDIT: - expression: object = arguments.get("expression") - if isinstance(expression, str): - result: SqlBindingResult = get_schema_validations( - requests=( - SqlSchemaValidationRequest( - sql=f"SELECT * FROM output WHERE {expression}", - dialect=profile.sql_analysis_dialect, - schema={"output": shape}, - known_functions=known_function_names(project.functions), - catalog=project.binding_catalog, - quoted_identifiers_ignore_case=profile.quoted_identifiers_ignore_case, - known_types=known_declared_types( - functions=project.functions, column_types=shapes - ), - ), - ) - )[0] - diagnostics.extend( - _model_error( - model=model, - code=diagnostic.code, - name=expression, - message=f"expression_is_true: {diagnostic.message}", - severity=DiagnosticSeverity(diagnostic.severity), - ) - for diagnostic in result.diagnostics - if diagnostic.code != BINDING_UNKNOWN_TABLE_INTERNAL_CODE - ) - elif audit.definition_name == _RELATIONSHIPS_AUDIT and column: - target: str = str(arguments.get("to", "")) - match: re.Match[str] | None = re.search( - r"__(?:ref|source|seed)\([\'\"]([^\'\"]+)[\'\"]\)", target - ) - target_shape: dict[str, str] | None = shapes.get(match.group(1) if match else target) - field: str = str(arguments.get("field", "")) - resolved_field: str | None = resolve_binding_column( - name=field, - columns=target_shape or {}, - dialect=profile.sql_analysis_dialect, - ignore_quoted_case=profile.quoted_identifiers_ignore_case, - ) - if target_shape and resolved_field is None: - diagnostics.append( - _model_error( - model=model, - code="B300", - name=field, - message=f"relationships target has no column '{field}'", - ) - ) - elif target_shape is not None and resolved_field is not None and column in shape: - diagnostics.extend( - _model_error( - model=model, - code="B301" - if diagnostic.severity == DiagnosticSeverity.ERROR - else diagnostic.code, - name=column, - message=f"relationships: {diagnostic.message}", - severity=DiagnosticSeverity(diagnostic.severity), - ) - for diagnostic in _comparison_result( - left=shape[column], - right=target_shape[resolved_field], - dialect=profile.sql_analysis_dialect, - ).diagnostics - ) - elif audit.definition_name == _ACCEPTED_VALUES_AUDIT and column in shape: - values: object = arguments.get("values") - if isinstance(values, (list, tuple)): - literals: list[str | None] = [_audit_literal(value) for value in values] - if literals and all(literal is not None for literal in literals): - result = _accepted_values_result( - literals=tuple(str(value) for value in literals), - dialect=profile.sql_analysis_dialect, - column_type=shape[column], - ) - diagnostics.extend( - _model_error( - model=model, - code="B301" if item.severity == DiagnosticSeverity.ERROR else item.code, - name=column, - message=f"accepted_values: {item.message}", - severity=DiagnosticSeverity(item.severity), - ) - for item in result.diagnostics - ) - return tuple(diagnostics) - - -@lru_cache(maxsize=256) -def _comparison_result(*, left: str, right: str, dialect: str | None) -> SqlBindingResult: - return get_schema_validations( - requests=( - SqlSchemaValidationRequest( - sql="SELECT lhs.value = rhs.value FROM lhs CROSS JOIN rhs", - dialect=dialect, - schema={"lhs": {"value": left}, "rhs": {"value": right}}, - ), - ) - )[0] - +from sqlbuild.compiler.compile.models import CompiledModelSqlTestPayload +from sqlbuild.compiler.compile.types import CompiledProjectView -def _audit_literal(value: object) -> str | None: - if value is None: - return "NULL" - if isinstance(value, bool): - return "TRUE" if value else "FALSE" - if isinstance(value, int | float): - return str(value) - if isinstance(value, str): - return "'" + value.replace("'", "''") + "'" - return None - - -@lru_cache(maxsize=256) -def _accepted_values_result( - *, literals: tuple[str, ...], dialect: str | None, column_type: str -) -> SqlBindingResult: - return get_schema_validations( - requests=( - SqlSchemaValidationRequest( - sql=f"SELECT value NOT IN ({', '.join(literals)}) FROM output", - dialect=dialect, - schema={"output": {"value": column_type}}, - ), - ) - )[0] - -def _sql_test_columns( - *, project: CompiledProject, profile: ExpressionInferenceProfile +def sql_test_columns( + *, project: CompiledProjectView, profile: ExpressionInferenceProfile ) -> dict[str, tuple[str, ...]]: """Infer fixture interfaces in one batch instead of repeating Python AST walks.""" bodies: dict[str, None] = {} @@ -203,49 +29,18 @@ def _sql_test_columns( ), profile=profile, ) - result: dict[str, tuple[str, ...]] = { - sql: _column_names(analysis) for sql, analysis in zip(queries, analyses, strict=True) - } - return result + return {sql: _column_names(analysis) for sql, analysis in zip(queries, analyses, strict=True)} def _column_names(analysis: NativeQueryColumns) -> tuple[str, ...]: return tuple(column.name for column in analysis.columns or ()) -def _names(value: object) -> tuple[str, ...]: +def names(value: object) -> tuple[str, ...]: + """The column names a metadata config value lists.""" + if isinstance(value, str): return (value,) if isinstance(value, (tuple, list)): return tuple(item for item in value if isinstance(item, str)) return () - - -def _model_error( - *, - model: CompiledModel, - code: str, - name: str, - message: str, - severity: DiagnosticSeverity = DiagnosticSeverity.ERROR, -) -> CompilerDiagnostic: - line: int - column: int - line, column = _text_position(text=model.authored_sql, name=name) - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=severity, - code=code, - message=message, - resource_type=CompiledResourceType.MODEL, - resource_name=model.name, - path=model.relative_path, - line=line, - column=column, - ) - - -def _text_position(*, text: str, name: str, offset: int = 0) -> tuple[int, int]: - match: re.Match[str] | None = re.search(rf"(? tuple[str, ...]: """Register authored names and the analysis-only scalar/table function names.""" names: set[str] = set() @@ -21,7 +21,7 @@ def known_function_names( def known_declared_types( *, - functions: tuple[CompiledFunction, ...] | tuple[CompileSqlFunctionInput, ...], + functions: tuple[CompiledFunctionView, ...] | tuple[FunctionInput, ...], column_types: dict[str, dict[str, str]], ) -> tuple[str, ...]: """Register types named by relation contracts and scalar/table function signatures.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 711b6e09ff..5c44a2f172 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Sequence from dataclasses import dataclass, replace from pathlib import Path from typing import Any, cast @@ -10,13 +9,15 @@ import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, +from sqlbuild.compiler.analysis_session.main._infer_native_query_column_rows import ( + infer_native_query_column_rows, ) -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeModelAnalysisRequest -from sqlbuild.compiler.compile._helpers.analysis.binding_requests import ( - get_complete_schema_binding_request, +from sqlbuild.compiler.analysis_session.models import ( + NativeColumnQuery, + NativeModelAnalyses, + NativeModelAnalysisRequest, ) +from sqlbuild.compiler.analysis_session.types import ColumnRow, DiagnosticRow from sqlbuild.compiler.compile._helpers.analysis.cache import ( build_analysis_cache_context, ) @@ -27,9 +28,8 @@ validate_hook_sql_syntax, validate_sql_syntax, ) -from sqlbuild.compiler.compile._helpers.assembly.binding_positions import ( - get_authored_binding_location, - query_line_offset, +from sqlbuild.compiler.compile._helpers.assembly.hand_built_models import ( + hand_built_compiled_model, ) from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( known_declared_types, @@ -42,9 +42,6 @@ from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( build_declared_column_types as _build_column_types_by_table, ) -from sqlbuild.compiler.compile._helpers.assembly.targets import ( - build_model_relation_target, -) from sqlbuild.compiler.compile._helpers.diagnostics.recovery import complete_semantic_diagnostics from sqlbuild.compiler.compile._helpers.diagnostics.scope import report_scope_index_errors from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( @@ -72,40 +69,28 @@ ) from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, - CompileAuditInput, - CompiledAudit, - CompiledFunction, - CompiledLineageColumnFact, - CompiledModel, CompiledObjectKey, CompiledProject, - CompiledRelationLocation, - CompiledSeed, - CompiledSource, CompiledSqlScenario, - CompileModelInput, CompileModelSqlTestInputPayload, - CompileProjectInputs, CompilerDiagnostic, - CompileSeedInput, - CompileSourceInput, - CompileSqlFunctionInput, CompileSqlScenarioInput, CompileSqlTestInput, DeclarationScopeResolver, - DynamicColumnContractProof, - InferredColumn, + HandBuiltAnalysis, MacroContext, - PolyglotAnalysisResult, -) -from sqlbuild.compiler.compile.models import ( - ModelSqlAnalysis as _ModelSqlAnalysis, ) from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, + CompiledAuditView, + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, + CompiledSeedView, + CompiledSourceView, + CompileInputsView, + ModelInput, + SourceInput, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.compiler.planner.types import ContractPolicy @@ -118,18 +103,13 @@ NativeProjectFacts, NativeProjectResources, ) -from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.compiler.sql_analysis.constants import ( - BINDING_UNKNOWN_TABLE_INTERNAL_CODE, NATIVE_DIALECT_ALIASES, NATIVE_FAILURE_PREFIX, ) from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case -from sqlbuild.compiler.sql_analysis.models import ( - SqlBindingDiagnostic, -) from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) @@ -138,8 +118,6 @@ ) from sqlbuild.spec.contracts.models import ( SchemaDynamicColumnFamily, - SourceEntry, - SourceLocation, ) @@ -151,13 +129,13 @@ class _DynamicContractAnalysisInputs: def assemble_compiled_project( *, - inputs: CompileProjectInputs, + inputs: CompileInputsView, inference_profile: ExpressionInferenceProfile | None = None, skip_column_inference: bool = False, column_lineage_mode: ColumnLineageMode = ColumnLineageMode.FAST, analysis_cache_dir: Path | None = None, analysis_model_names: frozenset[str] | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """Convert attached compile inputs into the planner-ready project view.""" sql_analysis_enabled: bool = ( @@ -202,7 +180,7 @@ def assemble_compiled_project( ) profile = replace(profile, binding_catalog=binding_catalog) if sql_analysis_enabled: - expression_sources: tuple[CompileSourceInput, ...] = tuple( + expression_sources: tuple[SourceInput, ...] = tuple( source_input for source_input in inputs.source_inputs if source_input.source_entry.expression @@ -262,14 +240,13 @@ def assemble_compiled_project( if sql_analysis_enabled and analysis_model_names != frozenset() else None ) - model_sql_analysis_by_name: dict[str, _ModelSqlAnalysis] = {} - native_session: Any | None = None + analyses: NativeModelAnalyses | None = None if sql_analysis_enabled: with ( record_compile_timing("model_analysis_ms"), record_compile_cpu_timing("model_analysis_cpu_ms"), ): - model_sql_analysis_by_name, native_session = analyze_model_sql( + analyses = analyze_model_sql( NativeModelAnalysisRequest( known_functions=known_function_names(inputs.sql_function_inputs), known_types=known_declared_types( @@ -294,11 +271,12 @@ def assemble_compiled_project( dynamic_families_by_table=dynamic_families_by_table, ) ) + native_session: Any | None = None if analyses is None else analyses.session native: NativeProjectFacts = project_facts_by_engine( inputs=inputs, dialect=profile.sql_analysis_dialect, analysis_model_names=analysis_model_names, - analyses=model_sql_analysis_by_name, + analyses=analyses, session=native_session, column_types_by_table=column_types_by_table, authoritative_column_types_by_table=complete_binding_schemas, @@ -323,13 +301,31 @@ def assemble_compiled_project( effective_target_name=inputs.effective_target_name, run_id=inputs.run_id, ) - project: CompiledProject = CompiledProject( + models: tuple[CompiledModelView, ...] = tuple( + _assemble_compiled_model( + model_input=model_input, + sql_analysis_enabled=( + sql_analysis_enabled + and model_input.sql_validation_enabled + and ( + analysis_model_names is None or _model_name(model_input) in analysis_model_names + ) + ), + sql_validation_enabled=( + analysis_model_names is None or _model_name(model_input) in analysis_model_names + ), + native_model=native.models[index], + column_nullability_by_table=column_nullability_by_table, + column_types_by_table=column_types_by_table, + dynamic_contract_analysis_inputs=dynamic_contract_analysis_inputs, + inference_profile=profile, + analyses=analyses, + allow_compact_analysis=allow_compact_analysis, + ) + for index, model_input in enumerate(inputs.model_inputs) + ) + fields: dict[str, Any] = dict( binding_catalog=profile.binding_catalog, - sql_expansions={ - model_input.model_file.file_path: model_input.sql_expansion - for model_input in inputs.model_inputs - if model_input.sql_expansion is not None - }, run_id=inputs.run_id, effective_target_name=inputs.effective_target_name, effective_connection=inputs.effective_connection, @@ -344,73 +340,11 @@ def assemble_compiled_project( project_config=inputs.project_config, local_config=inputs.local_config, ), - models=tuple( - _assemble_compiled_model( - model_input=model_input, - sql_analysis_enabled=( - sql_analysis_enabled - and model_input.sql_validation_enabled - and ( - analysis_model_names is None - or _model_name(model_input) in analysis_model_names - ) - ), - sql_validation_enabled=( - analysis_model_names is None or _model_name(model_input) in analysis_model_names - ), - native_model=native.models[index], - column_nullability_by_table=column_nullability_by_table, - column_types_by_table=column_types_by_table, - dynamic_contract_analysis_inputs=dynamic_contract_analysis_inputs, - inference_profile=profile, - sql_analysis=model_sql_analysis_by_name.get(model_input.model_file.file_path.stem), - allow_compact_analysis=allow_compact_analysis, - ) - for index, model_input in enumerate(inputs.model_inputs) - ), - sources=tuple( - _assemble_compiled_source( - source_input=source_input, - native_entry=native_resource( - resources=resources, - slot=SOURCE_SLOT, - items=resources.source_entries, - index=index, - ), - ) - for index, source_input in enumerate(inputs.source_inputs) - ), - seeds=tuple( - _assemble_compiled_seed( - seed_input=seed_input, - native_destination=native_resource( - resources=resources, - slot=SEED_SLOT, - items=resources.seed_destinations, - index=index, - ), - ) - for index, seed_input in enumerate(inputs.seed_inputs) - ), - functions=tuple( - _assemble_compiled_function( - function_input=function_input, - native_deps=resources.function_deps[index], - ) - for index, function_input in enumerate(inputs.sql_function_inputs) - ), - audits=tuple( - _assemble_compiled_audit( - audit_input=audit_input, - native_scope_deps=native_resource( - resources=resources, - slot=AUDIT_SLOT, - items=resources.audit_scope_deps, - index=index, - ), - ) - for index, audit_input in enumerate(inputs.audit_inputs) - ), + models=models, + sources=_compiled_sources(inputs=inputs, resources=resources), + seeds=_compiled_seeds(inputs=inputs, resources=resources), + functions=_compiled_functions(inputs=inputs, resources=resources), + audits=_compiled_audits(inputs=inputs, resources=resources), sql_tests=assemble_sql_tests_by_engine( inputs=inputs, macro_mock_queries=lambda test_input: _build_test_model_query_overrides( @@ -431,23 +365,37 @@ def assemble_compiled_project( loaded_macros=inputs.loaded_macros, diagnostics=( *inputs.diagnostics, - *_project_binding_diagnostics( - model_inputs=inputs.model_inputs, - analyses_by_name=model_sql_analysis_by_name, - dialect=profile.sql_analysis_dialect, - ), + *_project_binding_diagnostics(models=models, analyses=analyses), ), external_sql_reference_resolver=inputs.external_sql_reference_resolver, scope_index=scope_index, ) + project: CompiledProjectView = _assembled_project(inputs=inputs, fields=fields) return complete_semantic_diagnostics( project=project, profile=profile, resource_sql_analysis=sql_analysis_enabled and not inputs.no_sql_validation, native_session=native_session, - binding_results={ - name: analysis.polyglot_analysis.binding_diagnostics - for name, analysis in model_sql_analysis_by_name.items() + binding_results=_binding_rows(analyses), + ) + + +def _assembled_project(*, inputs: CompileInputsView, fields: dict[str, Any]) -> CompiledProjectView: + """The native owner's project view when it holds every model; else a `CompiledProject`.""" + + models: tuple[CompiledModelView, ...] = fields["models"] + if inputs.native_project is not None and all( + isinstance(model, _native.NativeCompiledModel) for model in models + ): + return inputs.native_project.project_view( + {**fields, "unmatched_literal_sql_relations": (), "native_session": None} + ) + return CompiledProject( + **fields, + sql_expansions={ + model_input.model_file.file_path: model_input.sql_expansion + for model_input in inputs.model_inputs + if model_input.sql_expansion is not None }, ) @@ -456,7 +404,7 @@ def _str_or_none(value: object) -> str | None: return value.strip() if isinstance(value, str) and value.strip() else None -def _connection_database_fallback(*, inputs: CompileProjectInputs) -> str | None: +def _connection_database_fallback(*, inputs: CompileInputsView) -> str | None: adapter_name: str = resolve_effective_adapter_name( project_config=inputs.project_config, local_config=inputs.local_config, @@ -468,7 +416,7 @@ def _connection_database_fallback(*, inputs: CompileProjectInputs) -> str | None def _assemble_compiled_model( *, - model_input: CompileModelInput, + model_input: ModelInput, sql_analysis_enabled: bool, sql_validation_enabled: bool = True, native_model: NativeModelFacts, @@ -476,9 +424,10 @@ def _assemble_compiled_model( column_types_by_table: dict[str, dict[str, str]] | None = None, dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs | None = None, inference_profile: ExpressionInferenceProfile | None = None, - sql_analysis: _ModelSqlAnalysis | None = None, + analyses: NativeModelAnalyses | None = None, allow_compact_analysis: bool = False, -) -> CompiledModel: +) -> CompiledModelView: + del column_types_by_table, dynamic_contract_analysis_inputs, allow_compact_analysis model_name: str = model_input.model_file.file_path.stem syntax_validated: bool = native_model.syntax_valid profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() @@ -486,13 +435,7 @@ def _assemble_compiled_model( sql=model_input.query_sql, cursor_type=model_input.config.values.get("cursor_type"), ) - inferred_columns: tuple[InferredColumn, ...] | None = None - fast_lineage_columns: Sequence[CompiledLineageColumnFact] | None = None - fast_lineage_has_star: bool = False - fast_lineage_star_resolved: bool = False - placeholders: dict[str, str] | None = ( - sql_analysis.placeholders if sql_analysis is not None else _model_placeholders(model_input) - ) + placeholders: dict[str, str] | None = _model_placeholders(model_input) if sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: for hook_name in ("pre_hooks", "post_hooks"): validate_hook_sql_syntax( @@ -503,19 +446,20 @@ def _assemble_compiled_model( placeholders=placeholders, dialect=profile.sql_analysis_dialect, ) + session_analysis: tuple[_native.NativeModelAnalysisSession, int] | None = None + fallback: list[ColumnRow] | None = None if sql_analysis_enabled: - if sql_analysis is None: + position: int | None = None if analyses is None else analyses.positions.get(model_name) + if analyses is None or position is None: raise _native.NativeCompilerError( f"{NATIVE_FAILURE_PREFIX}native model analysis: the session returned no " f"analysis for model '{model_name}'" ) - polyglot_analysis: PolyglotAnalysisResult = sql_analysis.polyglot_analysis - if polyglot_analysis.analysis_succeeded: - inferred_columns = polyglot_analysis.columns - fast_lineage_columns = polyglot_analysis.lineage_columns - fast_lineage_has_star = polyglot_analysis.has_star - fast_lineage_star_resolved = polyglot_analysis.star_resolved - else: + session_analysis = ( + cast(_native.NativeModelAnalysisSession, analyses.session), + position, + ) + if model_name not in analyses.succeeded: if model_input.sql_validation_enabled and not syntax_validated: validate_sql_syntax( query_sql=analysis_query_sql, @@ -524,7 +468,7 @@ def _assemble_compiled_model( placeholders=placeholders, dialect=profile.sql_analysis_dialect, ) - inferred_columns = infer_native_query_columns( + fallback = infer_native_query_column_rows( queries=( NativeColumnQuery( sql=analysis_query_sql, mode="parse", placeholders=placeholders @@ -532,7 +476,7 @@ def _assemble_compiled_model( ), profile=profile, column_nullability_by_table=column_nullability_by_table, - )[0].columns + )[0][1] elif sql_validation_enabled and model_input.sql_validation_enabled and not syntax_validated: validate_sql_syntax( query_sql=analysis_query_sql, @@ -541,54 +485,37 @@ def _assemble_compiled_model( placeholders=placeholders, dialect=profile.sql_analysis_dialect, ) - dynamic_column_contract: DynamicColumnContractProof | None = ( - sql_analysis.dynamic_column_contract - if sql_analysis is not None and sql_analysis.dynamic_column_contract is not None - else native_model.dynamic_contract + macro_deps: tuple[str, ...] = model_input.macro_deps or find_macro_call_names( + model_input.macro_source_sql ) - if dynamic_column_contract is not None and dynamic_column_contract.output_proven: - inferred_columns = dynamic_column_contract.fixed_columns - fast_lineage_has_star = True - return CompiledModel( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=model_name), + if isinstance(model_input, _native.NativeModelInput): + return model_input.compiled_model( + ( + native_model.deps, + session_analysis, + fallback, + native_model.dynamic_contract, + list(macro_deps), + ) + ) + return hand_built_compiled_model( + model_input=model_input, deps=native_model.deps, - name=model_name, - relative_path=model_input.model_file.relative_path, - query_sql=model_input.query_sql, - config=model_input.config, - destination=build_model_relation_target(model_input=model_input, model_name=model_name), - references=model_input.references, - schema_entry=model_input.schema_entry, - inferred_columns=inferred_columns, - fast_lineage_columns=fast_lineage_columns, - fast_lineage_has_star=fast_lineage_has_star, - fast_lineage_star_resolved=fast_lineage_star_resolved, - authored_sql=model_input.model_file.contents, - authored_query_sql=model_input.model_file.query_sql, - output_column_locations=model_input.model_file.output_column_locations, - extract_implicit_alias_columns=model_input.model_file.extract_implicit_alias_columns, - macro_deps=model_input.macro_deps or find_macro_call_names(model_input.macro_source_sql), - enum_declarations=model_input.enum_declarations, - constant_declarations=model_input.constant_declarations, - enum_columns=model_input.enum_columns, - binding_diagnostics=_binding_compiler_diagnostics( - model_input=model_input, - cleaned_sql=sql_analysis.cleaned_sql if sql_analysis is not None else None, - dialect=profile.sql_analysis_dialect, - diagnostics=(polyglot_analysis.binding_diagnostics if sql_analysis_enabled else ()), + analysis=HandBuiltAnalysis( + session=session_analysis, + fallback=fallback, + standalone_proof=native_model.dynamic_contract, ), - binding_validated=(polyglot_analysis.binding_validated if sql_analysis_enabled else False), - dynamic_column_contract=dynamic_column_contract, - rejected_sql_analysis_opt_out=model_input.rejected_sql_analysis_opt_out, + macro_deps=macro_deps, ) -def _model_name(model_input: CompileModelInput) -> str: +def _model_name(model_input: ModelInput) -> str: return model_input.model_file.file_path.stem def _build_dynamic_families_by_table( - inputs: CompileProjectInputs, + inputs: CompileInputsView, ) -> dict[str, tuple[SchemaDynamicColumnFamily, ...]]: return { model_input.schema_entry.name: model_input.schema_entry.dynamic_columns @@ -600,212 +527,107 @@ def _build_dynamic_families_by_table( def _project_binding_diagnostics( - *, - model_inputs: tuple[CompileModelInput, ...], - analyses_by_name: dict[str, _ModelSqlAnalysis], - dialect: str | None, + *, models: tuple[CompiledModelView, ...], analyses: NativeModelAnalyses | None ) -> tuple[CompilerDiagnostic, ...]: + """Every analysed model's located binding diagnostics, which the project also reports.""" + + if analyses is None: + return () diagnostics: list[CompilerDiagnostic] = [] - for model_input in model_inputs: - analysis: _ModelSqlAnalysis | None = analyses_by_name.get(_model_name(model_input)) - if analysis is None: - continue - diagnostics.extend( - _binding_compiler_diagnostics( - model_input=model_input, - cleaned_sql=analysis.cleaned_sql, - dialect=dialect, - diagnostics=analysis.polyglot_analysis.binding_diagnostics, - ) - ) + for model in models: + if model.name in analyses.positions: + diagnostics.extend(model.binding_diagnostics) return tuple(diagnostics) -def _binding_compiler_diagnostics( - *, - model_input: CompileModelInput, - diagnostics: tuple[SqlBindingDiagnostic, ...], - dialect: str | None, - cleaned_sql: str | None = None, -) -> tuple[CompilerDiagnostic, ...]: - line_offset: int | None = query_line_offset(model_input) - seen: set[tuple[str, str, int | None, int | None]] = set() - result: list[CompilerDiagnostic] = [] - for diagnostic in diagnostics: - if diagnostic.code == BINDING_UNKNOWN_TABLE_INTERNAL_CODE: - continue - if cleaned_sql is None: - cleaned_sql = get_complete_schema_binding_request( - query_sql=cursor_intrinsics_analysis_sql( - sql=model_input.query_sql, - cursor_type=model_input.config.values.get("cursor_type"), - ), - placeholders=_model_placeholders(model_input), - dialect=dialect, - binding_schema={}, - ).sql - line: int | None - column: int | None - location: SourceLocation | None = get_authored_binding_location( - path=model_input.model_file.relative_path, - authored_sql=model_input.model_file.contents, - authored_query_sql=model_input.model_file.query_sql, - cleaned_sql=cleaned_sql, - expansion=model_input.sql_expansion, - diagnostic=diagnostic, - ) - line, column = (location.line, location.column) if location is not None else (None, None) - if line is None: - line = ( - diagnostic.line + line_offset - if diagnostic.line is not None and line_offset is not None - else diagnostic.line - ) - column = diagnostic.column - key: tuple[str, str, int | None, int | None] = ( - diagnostic.code, - diagnostic.message, - line, - column, - ) - if ( - key in seen - and diagnostic.start is not None - and diagnostic.severity == DiagnosticSeverity.ERROR - ): - continue - seen.add(key) - result.append( - CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity(diagnostic.severity), - code=diagnostic.code, - message=diagnostic.message, - resource_type=CompiledResourceType.MODEL, - resource_name=_model_name(model_input), - path=model_input.model_file.relative_path, - line=line, - column=column, - location=location, - ) - ) - return tuple(result) - - -def _assemble_compiled_source( - *, source_input: CompileSourceInput, native_entry: SourceEntry -) -> CompiledSource: - return CompiledSource( - key=CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=native_entry.name), - deps=(), - name=native_entry.name, - source_entry=native_entry, - source_file=source_input.source_file, +def _binding_rows(analyses: NativeModelAnalyses | None) -> dict[str, list[DiagnosticRow]]: + """Each analysed model's raw binding rows, which semantic type recovery identifies.""" + + if analyses is None or analyses.session is None: + return {} + rows: list[list[DiagnosticRow]] = analyses.session.binding_rows() + return {name: rows[position] for name, position in analyses.positions.items()} + + +def _compiled_sources( + *, inputs: CompileInputsView, resources: NativeProjectResources +) -> tuple[CompiledSourceView, ...]: + """Each source's view over the compile's native project, raising assembly's error there.""" + + if not inputs.source_inputs: + return () + owner: _native.NativeCompiledProject = cast( + _native.NativeCompiledProject, inputs.native_project ) + owner.record_source_namespaces(list(resources.source_namespaces)) + sources: list[CompiledSourceView] = [] + for index in range(len(inputs.source_inputs)): + _ = native_resource( + resources=resources, slot=SOURCE_SLOT, items=resources.source_namespaces, index=index + ) + sources.append(owner.compiled_source(index)) + return tuple(sources) -def _assemble_compiled_seed( - *, seed_input: CompileSeedInput, native_destination: CompiledRelationLocation -) -> CompiledSeed: - return CompiledSeed( - key=CompiledObjectKey( - resource_type=CompiledResourceType.SEED, name=seed_input.schema_entry.name - ), - deps=(), - name=seed_input.schema_entry.name, - seed_file=seed_input.seed_file, - schema_entry=seed_input.schema_entry, - schema_file=seed_input.schema_file, - destination=native_destination, +def _compiled_seeds( + *, inputs: CompileInputsView, resources: NativeProjectResources +) -> tuple[CompiledSeedView, ...]: + """Each seed's view over the compile's native project, raising assembly's error at its seed.""" + + if not inputs.seed_inputs: + return () + owner: _native.NativeCompiledProject = cast( + _native.NativeCompiledProject, inputs.native_project ) + owner.record_seed_namespaces(list(resources.seed_namespaces)) + seeds: list[CompiledSeedView] = [] + for index in range(len(inputs.seed_inputs)): + _ = native_resource( + resources=resources, slot=SEED_SLOT, items=resources.seed_namespaces, index=index + ) + seeds.append(owner.compiled_seed(index)) + return tuple(seeds) -def _assemble_compiled_function( - *, - function_input: CompileSqlFunctionInput, - native_deps: tuple[CompiledObjectKey, ...], -) -> CompiledFunction: - return CompiledFunction( - key=CompiledObjectKey( - resource_type=CompiledResourceType( - function_node_type(return_columns=function_input.return_columns) - ), - name=function_input.name, - ), - deps=native_deps, - name=function_input.name, - relative_path=function_input.function_file.relative_path, - arguments=function_input.arguments, - returns=function_input.returns, - body_sql=function_input.body_sql, - return_columns=function_input.return_columns, - references=function_input.references, - destination=CompiledRelationLocation( - database=function_input.database, - schema=function_input.schema, - name=function_input.name, - qualified_name=None, - logical_database=function_input.logical_database, - logical_schema=function_input.logical_schema, - ), - fingerprint_destination=CompiledRelationLocation( - database=function_input.fingerprint_database, - schema=function_input.fingerprint_schema, - name=function_input.name, - qualified_name=None, - logical_database=function_input.fingerprint_logical_database, - logical_schema=function_input.fingerprint_logical_schema, - ), - language=function_input.language, - source_file_path=function_input.function_file.file_path, - runtime_version=function_input.runtime_version, - entry_point=function_input.entry_point, - packages=function_input.packages, - tags=function_input.tags, - description=function_input.description, +def _compiled_functions( + *, inputs: CompileInputsView, resources: NativeProjectResources +) -> tuple[CompiledFunctionView, ...]: + """Each function's view over the compile's native project, with its resolved deps.""" + + if not inputs.sql_function_inputs: + return () + owner: _native.NativeCompiledProject = cast( + _native.NativeCompiledProject, inputs.native_project ) + owner.record_function_deps(list(resources.function_deps)) + return tuple(owner.compiled_function(index) for index in range(len(inputs.sql_function_inputs))) -def _assemble_compiled_audit( - *, - audit_input: CompileAuditInput, - native_scope_deps: tuple[CompiledObjectKey, ...], -) -> CompiledAudit: - audit_name: str = _resolve_audit_name(audit_input) - normalized_target_kind: AttachedAuditTargetKind | None = None - if audit_input.attached_target_kind is not None: - normalized_target_kind = AttachedAuditTargetKind(audit_input.attached_target_kind) - return CompiledAudit( - key=CompiledObjectKey(resource_type=CompiledResourceType.AUDIT, name=audit_name), - scope_deps=native_scope_deps, - name=audit_name, - definition_name=(audit_input.audit_block.name or audit_input.audit_file.file_path.stem), - audit_file=audit_input.audit_file, - audit_block=audit_input.audit_block, - sql_body=audit_input.sql_body, - evaluation_mode=audit_input.evaluation_mode, - measurement_contract=audit_input.measurement_contract, - thresholds=audit_input.thresholds, - minimum_samples=audit_input.minimum_samples, - measure_sql=audit_input.measure_sql, - evidence_sql=audit_input.evidence_sql, - evidence_limit=audit_input.evidence_limit, - references=audit_input.references, - attached_target_kind=normalized_target_kind, - attached_target_name=audit_input.attached_target_name, - attached_column_name=audit_input.attached_column_name, - severity=audit_input.severity, - run_scope=audit_input.run_scope, - always_run=audit_input.always_run, - description=audit_input.description, +def _compiled_audits( + *, inputs: CompileInputsView, resources: NativeProjectResources +) -> tuple[CompiledAuditView, ...]: + """Each audit's view over the compile's native project, raising assembly's error there.""" + + if not inputs.audit_inputs: + return () + owner: _native.NativeCompiledProject = cast( + _native.NativeCompiledProject, inputs.native_project ) + owner.record_audit_scope_deps(list(resources.audit_scope_deps)) + audits: list[CompiledAuditView] = [] + for index in range(len(inputs.audit_inputs)): + _ = native_resource( + resources=resources, slot=AUDIT_SLOT, items=resources.audit_scope_deps, index=index + ) + audits.append(owner.compiled_audit(index)) + return tuple(audits) def _build_test_model_query_overrides( *, test_input: CompileSqlTestInput, - model_inputs: tuple[CompileModelInput, ...], - inputs: CompileProjectInputs, + model_inputs: tuple[ModelInput, ...], + inputs: CompileInputsView, ) -> dict[str, str]: """Re-expand each model from its pre-macro macro_source_sql with test macro mocks applied.""" @@ -834,7 +656,7 @@ def _build_test_model_query_overrides( ), ) overrides: dict[str, str] = {} - model_input: CompileModelInput + model_input: ModelInput for model_input in model_inputs: model_name: str = model_input.model_file.file_path.stem macro_source_sql: str = model_input.macro_source_sql or model_input.query_sql @@ -879,11 +701,3 @@ def _assemble_compiled_sql_scenario( source_path=scenario_input.scenario_file.relative_path, ownership_root=scenario_input.scenario_file.ownership_root, ) - - -def _resolve_audit_name(audit_input: CompileAuditInput) -> str: - if audit_input.name is not None: - return audit_input.name - if audit_input.audit_block.name is not None: - return audit_input.audit_block.name - return audit_input.audit_file.file_path.stem diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py index e1b7bfd68b..5282094dee 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py @@ -3,12 +3,12 @@ from __future__ import annotations from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.constants import TYPE_CHECKED_DIALECTS -def semantic_coverage(project: CompiledProject) -> dict[str, tuple[str, ...]]: +def semantic_coverage(project: CompiledProjectView) -> dict[str, tuple[str, ...]]: """Return partial-check reasons for machine reports.""" if not project.settings.sql_analysis: @@ -24,8 +24,8 @@ def semantic_coverage(project: CompiledProject) -> dict[str, tuple[str, ...]]: changed: bool = False for model in project.models: causes: set[str] = set(blocked_by.get(model.name, ())) - for reference in model.references: - causes.update(blocked_by.get(reference.ref_name, ())) + for _, name, _, _ in model.reference_rows: + causes.update(blocked_by.get(name, ())) if causes and causes != blocked_by.get(model.name): blocked_by[model.name] = causes changed = True @@ -37,23 +37,21 @@ def semantic_coverage(project: CompiledProject) -> dict[str, tuple[str, ...]]: reasons[model.name] = ("SQL analysis disabled",) continue partial: list[str] = [ - f"open source: {reference.ref_name}" - if reference.ref_kind == SqlReferenceKind.SOURCE - else f"open input: {reference.ref_name}" - for reference in model.references - if reference.ref_kind - in {SqlReferenceKind.SOURCE, SqlReferenceKind.REF, SqlReferenceKind.DBT_REF} - and reference.ref_name not in shapes + f"open source: {name}" if kind == SqlReferenceKind.SOURCE else f"open input: {name}" + for kind, name, _, _ in model.reference_rows + if kind in {SqlReferenceKind.SOURCE, SqlReferenceKind.REF, SqlReferenceKind.DBT_REF} + and name not in shapes ] partial.extend( f"depends on invalid model {name}" for name in sorted(blocked_by.get(model.name, set()) - {model.name}) ) - if not model.inferred_columns: + inferred: list[tuple[str, str | None, str]] | None = model.inferred_rows + if not inferred: partial.append("output shape could not be inferred") elif model.fast_lineage_has_star and model.name not in shapes: partial.append("unresolved star over open inputs") - if any(column.type is None for column in model.inferred_columns or ()): + if any(column_type is None for _, column_type, _ in inferred or ()): partial.append("some output types are unknown") if (project.sql_analysis_dialect or "generic") not in TYPE_CHECKED_DIALECTS: partial.append("type checking awaits dialect coercion support") diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py index b36912be17..63b904d861 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -12,12 +12,14 @@ from sqlbuild.compiler.compile._helpers.analysis.reference_names import table_function_analysis_name from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, - CompileModelInput, - CompileProjectInputs, CompileSqlReference, ) +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompileInputsView, + ModelInput, +) from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.planner.constants import ( SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, @@ -39,19 +41,27 @@ def binding_relation_names(references: tuple[CompileSqlReference, ...]) -> frozenset[str]: + return binding_relation_row_names( + [(str(reference.ref_kind), reference.ref_name, None, None) for reference in references] + ) + + +def binding_relation_row_names( + rows: list[tuple[str, str, str | None, int | None]], +) -> frozenset[str]: + """The analysis names `(kind, name, package, count)` reference rows bind, UDFs excluded.""" + return frozenset( - table_function_analysis_name(reference.ref_name) - if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION - else reference.ref_name - for reference in references - if reference.ref_kind != SqlReferenceKind.UDF + table_function_analysis_name(name) if kind == SqlReferenceKind.TABLE_FUNCTION else name + for kind, name, _, _ in rows + if kind != SqlReferenceKind.UDF ) -def binding_required_names(model_input: CompileModelInput) -> frozenset[str] | None: +def binding_required_names(model_input: ModelInput) -> frozenset[str] | None: if not model_input.sql_validation_enabled: return None - return binding_relation_names(model_input.references) + return binding_relation_row_names(model_input.reference_rows) def published_model_shape( @@ -76,7 +86,7 @@ def published_model_shape( def binding_schema_for_model( - *, model_input: CompileModelInput, complete_binding_schemas: dict[str, dict[str, str]] + *, model_input: ModelInput, complete_binding_schemas: dict[str, dict[str, str]] ) -> dict[str, dict[str, str]] | None: required_names: frozenset[str] | None = binding_required_names(model_input) if required_names is None: @@ -110,7 +120,7 @@ def inferred_binding_shape( ) -def build_declared_column_types(inputs: CompileProjectInputs) -> dict[str, dict[str, str]]: +def build_declared_column_types(inputs: CompileInputsView) -> dict[str, dict[str, str]]: """Collect declared type hints independently of closed-schema authority.""" facts: dict[str, dict[str, str]] = {} for model_input in inputs.model_inputs: @@ -135,7 +145,7 @@ def build_declared_column_types(inputs: CompileProjectInputs) -> dict[str, dict[ def build_column_nullability_by_table( - inputs: CompileProjectInputs, + inputs: CompileInputsView, ) -> dict[str, dict[str, InferredNullability]]: """Return what each model, seed, source and table function declares about column nulls.""" @@ -193,7 +203,7 @@ def _declared_column_nullability( return InferredNullability.UNKNOWN -def build_complete_binding_schemas(inputs: CompileProjectInputs) -> dict[str, dict[str, str]]: +def build_complete_binding_schemas(inputs: CompileInputsView) -> dict[str, dict[str, str]]: """Collect authoritative declared interfaces before SQL inference.""" schemas: dict[str, dict[str, str]] = {} for model in inputs.model_inputs: @@ -226,7 +236,7 @@ def build_complete_binding_schemas(inputs: CompileProjectInputs) -> dict[str, di def semantic_shapes( *, - project: CompiledProject, + project: CompiledProjectView, profile: ExpressionInferenceProfile | None = None, ) -> dict[str, dict[str, str]]: """Return closed shapes; absence means open, never an empty physical table.""" @@ -260,48 +270,33 @@ def semantic_shapes( for (source_name, _), source_shape in zip(expression_sources, source_shapes, strict=True): if source_shape is not None: shapes[source_name] = source_shape - pending: list[CompiledModel] = list(project.models) + pending: list[CompiledModelView] = list(project.models) while pending: - remaining: list[CompiledModel] = [] + remaining: list[CompiledModelView] = [] for model in pending: + declared: list[tuple[str, str | None]] | None = model.schema_column_rows + inferred: list[tuple[str, str | None, str]] | None = model.inferred_rows + rows: list[tuple[str, str, str | None, int | None]] = model.reference_rows + relations: frozenset[str] = binding_relation_row_names(rows) if ( model.config.values.get("contract") == ContractPolicy.ENFORCED - and model.schema_entry - and not model.schema_entry.dynamic_columns + and declared is not None + and not model.has_dynamic_columns ): + unchecked: frozenset[str] = model.unchecked_output_columns shapes[model.name] = { - column.name: "UNKNOWN" - if column.name in model.unchecked_output_columns - else column.type or "UNKNOWN" - for column in model.schema_entry.columns + name: "UNKNOWN" if name in unchecked else column_type or "UNKNOWN" + for name, column_type in declared } - elif model.inferred_columns and ( + elif inferred and ( not model.fast_lineage_has_star - or ( - model.fast_lineage_star_resolved - and model.references - and all( - ( - table_function_analysis_name(reference.ref_name) - if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION - else reference.ref_name - ) - in shapes - for reference in model.references - if reference.ref_kind != SqlReferenceKind.UDF - ) - ) + or (model.fast_lineage_star_resolved and rows and relations <= shapes.keys()) ): shapes[model.name] = published_model_shape( sql=model.query_sql, profile=replace(profile, binding_catalog=project.binding_catalog), - columns={ - column.name: column.type or "UNKNOWN" for column in model.inferred_columns - }, - inputs={ - name: shapes.get(name, {}) - for name in binding_relation_names(model.references) - }, + columns={name: column_type or "UNKNOWN" for name, column_type, _ in inferred}, + inputs={name: shapes.get(name, {}) for name in relations}, config_values=model.config.values, ) else: @@ -313,7 +308,7 @@ def semantic_shapes( def referenced_model_names( - *, model_input: CompileModelInput, available_names: frozenset[str] | None = None + *, model_input: ModelInput, available_names: frozenset[str] | None = None ) -> tuple[str, ...]: return tuple( sorted( @@ -328,7 +323,7 @@ def referenced_model_names( def upstream_signatures( - *, model_input: CompileModelInput, signatures: dict[str, str], available_names: frozenset[str] + *, model_input: ModelInput, signatures: dict[str, str], available_names: frozenset[str] ) -> dict[str, str]: """Return the output signatures of the analyzed models this model references.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py b/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py index 5fdc739a92..eb3c17763c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py @@ -22,7 +22,7 @@ known_function_names, ) from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( - binding_relation_names, + binding_relation_row_names, published_model_shape, semantic_shapes, ) @@ -30,13 +30,13 @@ cursor_intrinsics_analysis_sql, ) from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledSqlExpansion, CompilerDiagnostic, ) from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, CompiledResourceType, DiagnosticPhase, DiagnosticSeverity, @@ -54,14 +54,14 @@ def _model_shapes( - *, model: CompiledModel, shapes: dict[str, dict[str, str]] + *, model: CompiledModelView, shapes: dict[str, dict[str, str]] ) -> dict[str, dict[str, str]]: - return {name: shapes.get(name, {}) for name in binding_relation_names(model.references)} + return {name: shapes.get(name, {}) for name in binding_relation_row_names(model.reference_rows)} def get_source_binding_diagnostics( *, - project: CompiledProject, + project: CompiledProjectView, columns: dict[str, tuple[ColumnInfo, ...]], profile: ExpressionInferenceProfile, selected_keys: frozenset[CompiledObjectKey], @@ -98,7 +98,7 @@ def get_source_binding_diagnostics( else None ) profile = replace(profile, binding_catalog=physical_catalog) - by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} graph: dict[str, tuple[str, ...]] = {} for model in project.models: graph[model.name] = tuple( @@ -107,9 +107,9 @@ def get_source_binding_diagnostics( if reference.ref_kind == SqlReferenceKind.REF and reference.ref_name in by_name ) affected: set[str] = set() - models: list[CompiledModel] = [] + models: list[CompiledModelView] = [] for name in TopologicalSorter(graph).static_order(): - model: CompiledModel = by_name[name] + model: CompiledModelView = by_name[name] if model.key not in selected_keys or model.config.values.get("sql_analysis") is False: continue if any( @@ -217,7 +217,7 @@ def _adds_physical_evidence(*, physical: dict[str, str], known: dict[str, str] | ) -def _placeholders(model: CompiledModel) -> dict[str, str] | None: +def _placeholders(model: CompiledModelView) -> dict[str, str] | None: values: object = model.config.values.get("placeholders") if isinstance(values, dict): return {str(key): str(value) for key, value in values.items()} diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py b/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py index 8912228100..bf4775ec49 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/targets.py @@ -2,11 +2,12 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledRelationLocation, CompileModelInput +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.compile.types import ModelInput def build_model_relation_target( - *, model_input: CompileModelInput, model_name: str + *, model_input: ModelInput, model_name: str ) -> CompiledRelationLocation: """Resolve one model's configured physical and logical destination.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py deleted file mode 100644 index 4f063906f9..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ /dev/null @@ -1,1028 +0,0 @@ -"""Attachment helpers for building pre-semantic compile inputs.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from pathlib import Path - -from sqlbuild.compiler.attachments.constants import DEFAULT_POLICY, INSTANCE_POLICY -from sqlbuild.compiler.attachments.main._render_native_attached_audit import ( - render_native_attached_audit, -) -from sqlbuild.compiler.attachments.models import NativeAuditPolicies, NativeRenderedAudit -from sqlbuild.compiler.auditing.constants import ( - MEASUREMENT_MINIMUM_SAMPLES_HEADER_KEY, - MEASUREMENT_THRESHOLDS_HEADER_KEY, -) -from sqlbuild.compiler.auditing.main._builtins import build_builtin_audit_resolution -from sqlbuild.compiler.auditing.main._parse_audit_instance import ( - parse_measurement_thresholds, - parse_minimum_samples, -) -from sqlbuild.compiler.auditing.models import MeasurementThresholds -from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditRunScope, AuditSeverity -from sqlbuild.compiler.compile._helpers.attachment.references import ( - build_known_ref_names, - build_known_seed_names, - build_known_source_names, - validate_audit_references, -) -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( - merge_call_site_references, -) -from sqlbuild.compiler.compile._helpers.named_declarations.core import ( - declaration_file_expansion, - named_declaration_usages, -) -from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics -from sqlbuild.compiler.compile._helpers.render.sql_vars import ( - expand_authored_sql_result, -) -from sqlbuild.compiler.compile.constants import ( - SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - AuthoredSqlExpansionResult, - CompileAuditInput, - CompileModelInput, - CompilerDiagnostic, - CompileSeedInput, - CompileSourceInput, - CompileSqlReference, - DeclarationExpansionContext, - LoadedMacro, - MacroContext, - ModelInputBuildContext, - SqlReferenceOrigin, -) -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditBlock, - DiscoveredAuditFile, - DiscoveredProjectInputs, -) -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import ( - SchemaAuditInstance, - SchemaColumn, - SettingsConfig, - SourceColumnEntry, -) - -_CROSS_MODEL_MINIMUM: int = 2 - - -@dataclass(frozen=True) -class _AuditConsumer: - """The resource that attaches a generic audit, used for scope visibility and placement.""" - - identity: ResourceIdentity - path: Path - - -@dataclass(frozen=True) -class _AuditAttachmentContext: - """Run-constant inputs shared across attached audit rendering.""" - - generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] - loaded_macros: dict[str, LoadedMacro] - known_model_names: set[str] - known_seed_names: set[str] - known_source_names: set[str] - default_audit_severity: AuditSeverity | None - default_audit_run_scope: str | None - effective_vars: dict[str, object] - macro_context: MacroContext - declaration_expansion: DeclarationExpansionContext - sql_lexical_syntax: SqlLexicalSyntax - scoped_declarations: dict[tuple[Path, DeclarationIdentity], DeclarationExpansionContext] = ( - field(default_factory=dict, compare=False, repr=False) - ) - - def remember_scoped_declarations( - self, - *, - key: tuple[Path, DeclarationIdentity], - declarations: DeclarationExpansionContext, - ) -> None: - """Retain one resolved audit scope for reuse in this compile invocation.""" - - self.scoped_declarations[key] = declarations - - -def build_project_audit_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - model_inputs: tuple[CompileModelInput, ...], - source_inputs: tuple[CompileSourceInput, ...], - seed_inputs: tuple[CompileSeedInput, ...], -) -> tuple[tuple[CompileAuditInput, ...], tuple[CompilerDiagnostic, ...]]: - """Resolve built-in-aware generic audits and build every project audit input.""" - - generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] - diagnostics: tuple[CompilerDiagnostic, ...] - generic_audit_definitions, diagnostics = build_builtin_audit_resolution( - index_generic_audit_definitions(discovered_inputs.audit_files) - ) - audit_inputs: tuple[CompileAuditInput, ...] = build_audit_inputs( - discovered_inputs=discovered_inputs, - context=context, - model_inputs=model_inputs, - source_inputs=source_inputs, - generic_audit_definitions=generic_audit_definitions, - seed_inputs=seed_inputs, - ) - return audit_inputs, diagnostics - - -def build_audit_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - model_inputs: tuple[CompileModelInput, ...], - source_inputs: tuple[CompileSourceInput, ...], - generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] - | None = None, - seed_inputs: tuple[CompileSeedInput, ...] = (), -) -> tuple[CompileAuditInput, ...]: - """Build compile-time audit inputs from discovered SQL audit blocks.""" - - effective_settings: SettingsConfig = context.effective_settings - effective_vars: dict[str, object] = context.effective_vars - macro_context: MacroContext = context.macro_context - loaded_macros: dict[str, LoadedMacro] = context.loaded_macros - declaration_expansion: DeclarationExpansionContext = context.declaration_expansion - sql_lexical_syntax: SqlLexicalSyntax = context.sql_lexical_syntax - known_model_names: set[str] = build_known_ref_names(discovered_inputs) - known_seed_names: set[str] = build_known_seed_names(discovered_inputs) - known_source_names: set[str] = build_known_source_names(discovered_inputs) - if generic_audit_definitions is None: - generic_audit_definitions = index_generic_audit_definitions(discovered_inputs.audit_files) - default_audit_severity: AuditSeverity | None = effective_settings.default_audit_severity - default_audit_run_scope: str | None = effective_settings.default_audit_run_scope - attachment_context: _AuditAttachmentContext = _AuditAttachmentContext( - generic_audit_definitions=generic_audit_definitions, - loaded_macros=loaded_macros, - known_model_names=known_model_names, - known_seed_names=known_seed_names, - known_source_names=known_source_names, - default_audit_severity=default_audit_severity, - default_audit_run_scope=default_audit_run_scope, - effective_vars=effective_vars, - macro_context=macro_context, - declaration_expansion=declaration_expansion, - sql_lexical_syntax=sql_lexical_syntax, - ) - audit_inputs: list[CompileAuditInput] = list( - _build_singular_audit_inputs( - discovered_inputs=discovered_inputs, context=attachment_context - ) - ) - model_input: CompileModelInput - for model_input in model_inputs: - if model_input.schema_entry is None: - continue - audit_inputs.extend( - build_model_attached_audit_inputs(model_input=model_input, context=attachment_context) - ) - audit_inputs.extend( - _build_source_and_seed_audit_inputs( - source_inputs=source_inputs, seed_inputs=seed_inputs, context=attachment_context - ) - ) - return tuple(audit_inputs) - - -def _build_singular_audit_inputs( - *, discovered_inputs: DiscoveredProjectInputs, context: _AuditAttachmentContext -) -> tuple[CompileAuditInput, ...]: - attachment_context: _AuditAttachmentContext = context - effective_vars: dict[str, object] = context.effective_vars - macro_context: MacroContext = context.macro_context - loaded_macros: dict[str, LoadedMacro] = context.loaded_macros - sql_lexical_syntax: SqlLexicalSyntax = context.sql_lexical_syntax - known_model_names: set[str] = context.known_model_names - known_seed_names: set[str] = context.known_seed_names - known_source_names: set[str] = context.known_source_names - default_audit_severity: AuditSeverity | None = context.default_audit_severity - default_audit_run_scope: str | None = context.default_audit_run_scope - audit_inputs: list[CompileAuditInput] = [] - audit_file: DiscoveredAuditFile - for audit_file in discovered_inputs.audit_files: - if is_generic_audit_file(audit_file): - continue - audit_block: DiscoveredAuditBlock - for audit_block in audit_file.blocks: - audit_identity: DeclarationIdentity = DeclarationIdentity( - DeclarationKind.SINGULAR_AUDIT, - audit_block.name or audit_file.relative_path.stem, - ) - scoped_declarations: DeclarationExpansionContext = _scoped_audit_declarations( - context=attachment_context, - file_path=audit_file.file_path, - consumer=audit_identity, - ) - expansion: AuthoredSqlExpansionResult = expand_authored_sql_result( - sql=audit_block.sql_body, - file_path=audit_file.file_path, - effective_vars=effective_vars, - loaded_macros=loaded_macros, - macro_context=macro_context, - declarations=scoped_declarations.declarations, - declaration_resolver=scoped_declarations.resolver, - value_renderer=scoped_declarations.value_renderer, - collection_rendering=scoped_declarations.collection_rendering, - ) - expanded_sql_body: str = expansion.sql - evidence_expansion: AuthoredSqlExpansionResult | None = None - expanded_evidence_sql: str | None = None - if audit_block.evidence_sql is not None: - evidence_expansion = expand_authored_sql_result( - sql=audit_block.evidence_sql, - file_path=audit_file.file_path, - effective_vars=effective_vars, - loaded_macros=loaded_macros, - macro_context=macro_context, - declarations=scoped_declarations.declarations, - declaration_resolver=scoped_declarations.resolver, - value_renderer=scoped_declarations.value_renderer, - collection_rendering=scoped_declarations.collection_rendering, - ) - expanded_evidence_sql = evidence_expansion.sql - reject_cursor_intrinsics( - sql=expanded_sql_body, - context=f"Audit '{audit_block.name or audit_file.file_path.stem}'", - ) - if expanded_evidence_sql is not None: - reject_cursor_intrinsics( - sql=expanded_evidence_sql, - context=f"Audit '{audit_block.name or audit_file.file_path.stem}' evidence", - ) - references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=_combined_references( - expanded_sql_body, - expanded_evidence_sql, - syntax=sql_lexical_syntax, - origin=_audit_reference_origin(audit_file), - ), - argument_references=_argument_references(expansion, evidence_expansion), - ) - validate_audit_references( - references=references, - audit_file=audit_file, - known_model_names=known_model_names, - known_seed_names=known_seed_names, - known_source_names=known_source_names, - ) - referenced_resources: tuple[ResourceIdentity, ...] = singular_audit_resources( - references=references, - audit_file=audit_file, - audit_name=audit_identity.name, - ) - header_severity: str | None = _str_from_dict( - values=audit_block.header_values, key="severity" - ) - header_run_scope: str | None = _str_from_dict( - values=audit_block.header_values, key="run_scope" - ) - header_always_run: bool = _bool_from_dict( - values=audit_block.header_values, key="always_run" - ) - thresholds: MeasurementThresholds | None = parse_measurement_thresholds( - raw_value=audit_block.header_values.get("thresholds"), - file_path=audit_file.relative_path, - label="standalone measurement audit", - error_class=CompileInputError, - ) - minimum_samples: int | None = parse_minimum_samples( - raw_value=audit_block.header_values.get("minimum_samples"), - file_path=audit_file.relative_path, - label="standalone measurement audit", - error_class=CompileInputError, - ) - if audit_block.evaluation_mode == AuditEvaluationMode.MEASUREMENT: - if header_severity is not None: - raise CompileInputError( - f"{audit_file.relative_path}: measurement audits must not define severity; " - "severity derives from thresholds" - ) - if thresholds is None: - raise CompileInputError( - f"{audit_file.relative_path}: standalone measurement audit must define " - "thresholds in its AUDIT header" - ) - resolved_severity: AuditSeverity = measurement_policy_severity(thresholds) - else: - resolved_severity = resolve_audit_severity( - instance_severity=header_severity, - default_severity=default_audit_severity, - audit_label=str(audit_file.relative_path), - ) - resolved_run_scope: str = resolve_audit_run_scope( - instance_run_scope=header_run_scope, - default_run_scope=default_audit_run_scope, - ) - audit_inputs.append( - CompileAuditInput( - audit_file=audit_file, - audit_block=audit_block, - sql_body=expanded_sql_body, - evaluation_mode=audit_block.evaluation_mode, - measurement_contract=audit_block.measurement_contract, - thresholds=thresholds, - minimum_samples=minimum_samples, - measure_sql=( - expanded_sql_body - if audit_block.evaluation_mode == AuditEvaluationMode.MEASUREMENT - else None - ), - evidence_sql=expanded_evidence_sql, - references=references, - severity=resolved_severity, - run_scope=resolved_run_scope, - always_run=header_always_run, - declaration_usages=( - expansion.usages - + (() if evidence_expansion is None else evidence_expansion.usages) - + tuple( - UsageRecord(consumer=resource, declaration=audit_identity) - for resource in referenced_resources - ) - ), - ) - ) - return tuple(audit_inputs) - - -def _build_source_and_seed_audit_inputs( - *, - source_inputs: tuple[CompileSourceInput, ...], - seed_inputs: tuple[CompileSeedInput, ...], - context: _AuditAttachmentContext, -) -> tuple[CompileAuditInput, ...]: - audit_inputs: list[CompileAuditInput] = [] - source_input: CompileSourceInput - for source_input in source_inputs: - audit_inputs.extend( - build_source_attached_audit_inputs( - source_input=source_input, - context=context, - ) - ) - seed_input: CompileSeedInput - for seed_input in seed_inputs: - audit_inputs.extend( - build_seed_attached_audit_inputs(seed_input=seed_input, context=context) - ) - return tuple(audit_inputs) - - -def build_model_attached_audit_inputs( - *, - model_input: CompileModelInput, - context: _AuditAttachmentContext, -) -> tuple[CompileAuditInput, ...]: - """Render schema-attached model audits into compile audit inputs.""" - - if model_input.schema_entry is None: - raise CompileInputError( - f"Model file {model_input.model_file.relative_path} has no schema entry for " - "schema-attached audits" - ) - owner_file: Path = ( - model_input.schema_file.relative_path - if model_input.schema_file is not None - else model_input.model_file.relative_path - ) - model_consumer: _AuditConsumer = _AuditConsumer( - identity=ResourceIdentity(ResourceKind.MODEL, model_input.model_file.file_path.stem), - path=model_input.model_file.relative_path, - ) - attached_audit_inputs: list[CompileAuditInput] = [] - audit_instance: SchemaAuditInstance - for audit_instance in model_input.schema_entry.audits: - attached_column_name: str | None = _explicit_audit_column_name( - audit_instance=audit_instance, - ) - attached_audit_inputs.append( - build_attached_audit_input( - audit_instance=audit_instance, - owner_file=owner_file, - implicit_arguments={ - "model": model_input.model_file.file_path.stem, - "relation": SqlReferenceKind.REF.example_call( - model_input.model_file.file_path.stem, - quote='"', - ), - }, - attached_target_kind=AttachedAuditTargetKind.MODEL, - attached_target_name=model_input.model_file.file_path.stem, - attached_column_name=attached_column_name, - context=context, - consumer=model_consumer, - ) - ) - column_entry: SchemaColumn - for column_entry in model_input.schema_entry.columns: - column_owner_file: Path = ( - column_entry.location.path if column_entry.location is not None else owner_file - ) - for audit_instance in column_entry.audits: - audit_owner_file: Path = ( - audit_instance.location.path - if audit_instance.location is not None - else column_owner_file - ) - attached_audit_inputs.append( - build_attached_audit_input( - audit_instance=audit_instance, - owner_file=audit_owner_file, - implicit_arguments={ - "model": model_input.model_file.file_path.stem, - "relation": SqlReferenceKind.REF.example_call( - model_input.model_file.file_path.stem, - quote='"', - ), - "column": column_entry.name, - }, - attached_target_kind=AttachedAuditTargetKind.MODEL, - attached_target_name=model_input.model_file.file_path.stem, - attached_column_name=column_entry.name, - context=context, - consumer=model_consumer, - ) - ) - return tuple(attached_audit_inputs) - - -def build_source_attached_audit_inputs( - *, - source_input: CompileSourceInput, - context: _AuditAttachmentContext, -) -> tuple[CompileAuditInput, ...]: - """Render source-attached audits into compile audit inputs.""" - - source_consumer: _AuditConsumer = _AuditConsumer( - identity=ResourceIdentity(ResourceKind.SOURCE, source_input.source_entry.name), - path=source_input.source_file.relative_path, - ) - attached_audit_inputs: list[CompileAuditInput] = [] - audit_instance: SchemaAuditInstance - for audit_instance in source_input.source_entry.audits: - attached_audit_inputs.append( - build_attached_audit_input( - audit_instance=audit_instance, - owner_file=source_input.source_file.relative_path, - implicit_arguments={ - "source": source_input.source_entry.name, - "relation": SqlReferenceKind.SOURCE.example_call( - source_input.source_entry.name, - quote='"', - ), - }, - attached_target_kind=AttachedAuditTargetKind.SOURCE, - attached_target_name=source_input.source_entry.name, - attached_column_name=None, - context=context, - consumer=source_consumer, - ) - ) - column_entry: SourceColumnEntry - for column_entry in source_input.source_entry.columns: - for audit_instance in column_entry.audits: - attached_audit_inputs.append( - build_attached_audit_input( - audit_instance=audit_instance, - owner_file=source_input.source_file.relative_path, - implicit_arguments={ - "source": source_input.source_entry.name, - "relation": SqlReferenceKind.SOURCE.example_call( - source_input.source_entry.name, - quote='"', - ), - "column": column_entry.name, - }, - attached_target_kind=AttachedAuditTargetKind.SOURCE, - attached_target_name=source_input.source_entry.name, - attached_column_name=column_entry.name, - context=context, - consumer=source_consumer, - ) - ) - return tuple(attached_audit_inputs) - - -def build_seed_attached_audit_inputs( - *, - seed_input: CompileSeedInput, - context: _AuditAttachmentContext, -) -> tuple[CompileAuditInput, ...]: - """Render seed-attached table and column audits into compile audit inputs.""" - - seed_name: str = seed_input.schema_entry.name - seed_consumer: _AuditConsumer = _AuditConsumer( - identity=ResourceIdentity(ResourceKind.SEED, seed_name), - path=seed_input.seed_file.relative_path, - ) - implicit_arguments: dict[str, str] = { - "seed": seed_name, - "relation": SqlReferenceKind.SEED.example_call(seed_name, quote='"'), - } - attached_audit_inputs: list[CompileAuditInput] = [ - build_attached_audit_input( - audit_instance=audit_instance, - owner_file=seed_input.schema_file.relative_path, - implicit_arguments=implicit_arguments, - attached_target_kind=AttachedAuditTargetKind.SEED, - attached_target_name=seed_name, - attached_column_name=_explicit_audit_column_name(audit_instance=audit_instance), - context=context, - consumer=seed_consumer, - ) - for audit_instance in seed_input.schema_entry.audits - ] - column_entry: SchemaColumn - for column_entry in seed_input.schema_entry.columns: - attached_audit_inputs.extend( - build_attached_audit_input( - audit_instance=audit_instance, - owner_file=seed_input.schema_file.relative_path, - implicit_arguments={**implicit_arguments, "column": column_entry.name}, - attached_target_kind=AttachedAuditTargetKind.SEED, - attached_target_name=seed_name, - attached_column_name=column_entry.name, - context=context, - consumer=seed_consumer, - ) - for audit_instance in column_entry.audits - ) - return tuple(attached_audit_inputs) - - -def _explicit_audit_column_name( - *, - audit_instance: SchemaAuditInstance, -) -> str | None: - raw_column: object | None = audit_instance.arguments.get("column") - return raw_column if isinstance(raw_column, str) else None - - -def _scoped_audit_declarations( - *, context: _AuditAttachmentContext, file_path: Path, consumer: DeclarationIdentity -) -> DeclarationExpansionContext: - key: tuple[Path, DeclarationIdentity] = (file_path, consumer) - cached: DeclarationExpansionContext | None = context.scoped_declarations.get(key) - if cached is not None: - return cached - resolved: DeclarationExpansionContext = declaration_file_expansion( - context=context.declaration_expansion, - file_path=file_path, - consumer=consumer, - ) - context.remember_scoped_declarations(key=key, declarations=resolved) - return resolved - - -def build_attached_audit_input( - *, - audit_instance: SchemaAuditInstance, - owner_file: Path, - implicit_arguments: dict[str, str], - attached_target_kind: str, - attached_target_name: str, - attached_column_name: str | None, - context: _AuditAttachmentContext, - consumer: _AuditConsumer, -) -> CompileAuditInput: - """Render one attached generic audit instance into a compile audit input.""" - - definition: tuple[DiscoveredAuditFile, DiscoveredAuditBlock] | None = ( - context.generic_audit_definitions.get(audit_instance.definition_name) - ) - if definition is None: - raise CompileInputError( - f"{owner_file} references unknown generic audit '{audit_instance.definition_name}'" - ) - attachment_usages: tuple[UsageRecord, ...] = named_declaration_usages( - resolver=context.declaration_expansion.resolver, - kind=DeclarationKind.AUDIT, - name=audit_instance.definition_name, - consumer=consumer.identity, - consumer_path=consumer.path, - ) - evaluation_mode: AuditEvaluationMode = definition[1].evaluation_mode - if evaluation_mode == AuditEvaluationMode.MEASUREMENT: - if audit_instance.severity is not None: - raise CompileInputError( - f"{owner_file} audit '{audit_instance.definition_name}': measurement audit " - "attachments must not define severity; severity derives from thresholds" - ) - if audit_instance.thresholds is None: - raise CompileInputError( - f"{owner_file} audit '{audit_instance.definition_name}': measurement audit " - "attachments must define at least one threshold" - ) - elif audit_instance.thresholds is not None or audit_instance.minimum_samples is not None: - raise CompileInputError( - f"{owner_file} audit '{audit_instance.definition_name}': thresholds and " - "minimum_samples are only valid for measurement audits" - ) - native_rendering: NativeRenderedAudit = render_native_attached_audit( - labels=(str(owner_file), audit_instance.definition_name), - sql_body=definition[1].sql_body, - evidence_sql=definition[1].evidence_sql, - implicit_arguments=implicit_arguments, - explicit_arguments=audit_instance.arguments, - policies=_native_audit_policies(audit_instance=audit_instance, context=context), - ) - if native_rendering.render_error is not None: - raise CompileInputError(native_rendering.render_error) - rendered_sql_body: str = native_rendering.sql_body - rendered_evidence_sql: str | None = native_rendering.evidence_sql - scoped_declarations: DeclarationExpansionContext = _scoped_audit_declarations( - context=context, - file_path=definition[0].file_path, - consumer=DeclarationIdentity(DeclarationKind.AUDIT, audit_instance.definition_name), - ) - expansion: AuthoredSqlExpansionResult = expand_authored_sql_result( - sql=rendered_sql_body, - file_path=definition[0].file_path, - effective_vars=context.effective_vars, - loaded_macros=context.loaded_macros, - macro_context=context.macro_context, - declarations=scoped_declarations.declarations, - declaration_resolver=scoped_declarations.resolver, - value_renderer=scoped_declarations.value_renderer, - collection_rendering=scoped_declarations.collection_rendering, - ) - expanded_sql_body: str = expansion.sql - evidence_expansion: AuthoredSqlExpansionResult | None = None - expanded_evidence_sql: str | None = None - if rendered_evidence_sql is not None: - evidence_expansion = expand_authored_sql_result( - sql=rendered_evidence_sql, - file_path=definition[0].file_path, - effective_vars=context.effective_vars, - loaded_macros=context.loaded_macros, - macro_context=context.macro_context, - declarations=scoped_declarations.declarations, - declaration_resolver=scoped_declarations.resolver, - value_renderer=scoped_declarations.value_renderer, - collection_rendering=scoped_declarations.collection_rendering, - ) - expanded_evidence_sql = evidence_expansion.sql - reject_cursor_intrinsics( - sql=expanded_sql_body, - context=f"Audit '{audit_instance.definition_name}'", - ) - if expanded_evidence_sql is not None: - reject_cursor_intrinsics( - sql=expanded_evidence_sql, - context=f"Audit '{audit_instance.definition_name}' evidence", - ) - references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=_combined_references( - expanded_sql_body, - expanded_evidence_sql, - syntax=context.sql_lexical_syntax, - origin=_audit_reference_origin(definition[0]), - ), - argument_references=_argument_references(expansion, evidence_expansion), - ) - validate_audit_references( - references=references, - audit_file=definition[0], - known_model_names=context.known_model_names, - known_seed_names=context.known_seed_names, - known_source_names=context.known_source_names, - ) - audit_label: str = f"{owner_file} audit '{audit_instance.definition_name}'" - resolved_severity: AuditSeverity - resolved_run_scope: str - resolved_severity, resolved_run_scope = _native_audit_policy_values( - rendering=native_rendering, audit_instance=audit_instance, context=context - ) - validate_model_attached_audit_references( - references=references, - attached_target_kind=attached_target_kind, - attached_target_name=attached_target_name, - audit_label=audit_label, - ) - return CompileAuditInput( - audit_file=definition[0], - audit_block=definition[1], - sql_body=expanded_sql_body, - evaluation_mode=evaluation_mode, - measurement_contract=definition[1].measurement_contract, - thresholds=audit_instance.thresholds, - minimum_samples=audit_instance.minimum_samples, - measure_sql=( - expanded_sql_body if evaluation_mode == AuditEvaluationMode.MEASUREMENT else None - ), - evidence_sql=expanded_evidence_sql, - evidence_limit=audit_instance.evidence_limit, - name=audit_instance.name, - description=audit_instance.description, - references=references, - attached_target_kind=attached_target_kind, - attached_target_name=attached_target_name, - attached_column_name=attached_column_name, - severity=resolved_severity, - run_scope=resolved_run_scope, - always_run=audit_instance.always_run, - declaration_usages=( - attachment_usages - + expansion.usages - + (() if evidence_expansion is None else evidence_expansion.usages) - ), - ) - - -def _native_audit_policies( - *, audit_instance: SchemaAuditInstance, context: _AuditAttachmentContext -) -> NativeAuditPolicies: - thresholds: MeasurementThresholds | None = audit_instance.thresholds - return NativeAuditPolicies( - has_thresholds=thresholds is not None, - threshold_error=thresholds is not None and thresholds.error is not None, - instance_severity=audit_instance.severity, - default_severity=context.default_audit_severity, - instance_run_scope=audit_instance.run_scope, - default_run_scope=context.default_audit_run_scope, - ) - - -def _native_audit_policy_values( - *, - rendering: NativeRenderedAudit, - audit_instance: SchemaAuditInstance, - context: _AuditAttachmentContext, -) -> tuple[AuditSeverity, str]: - run_scope: str | None = ( - audit_instance.run_scope - if rendering.run_scope_source == INSTANCE_POLICY - else context.default_audit_run_scope - if rendering.run_scope_source == DEFAULT_POLICY - else None - ) - if rendering.policy_error is not None: - raise CompileInputError(rendering.policy_error) - return AuditSeverity(rendering.severity), ( - AuditRunScope.DELTA_AND_FINAL if run_scope is None else run_scope - ) - - -def index_generic_audit_definitions( - audit_files: tuple[DiscoveredAuditFile, ...], -) -> dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]]: - """Index project-wide and scoped generic audit definitions by name.""" - - definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] = {} - audit_file: DiscoveredAuditFile - for audit_file in audit_files: - if not is_generic_audit_file(audit_file): - continue - if len(audit_file.blocks) != 1: - raise CompileInputError( - f"Generic audit definition {audit_file.relative_path} must contain exactly " - "one AUDIT block" - ) - block: DiscoveredAuditBlock = audit_file.blocks[0] - if block.evaluation_mode == AuditEvaluationMode.MEASUREMENT and ( - MEASUREMENT_THRESHOLDS_HEADER_KEY in block.header_values - or MEASUREMENT_MINIMUM_SAMPLES_HEADER_KEY in block.header_values - ): - raise CompileInputError( - f"Generic measurement audit definition {audit_file.relative_path} must not " - "define thresholds or minimum_samples in its AUDIT header; attachment owns policy" - ) - definition_name: str = audit_file.file_path.stem - if definition_name in definitions: - raise CompileInputError( - f"Duplicate generic audit definition found for '{definition_name}' in " - f"{definitions[definition_name][0].relative_path} and {audit_file.relative_path}" - ) - definitions[definition_name] = (audit_file, audit_file.blocks[0]) - return definitions - - -def is_generic_audit_file(audit_file: DiscoveredAuditFile) -> bool: - """Return whether a discovered audit file is a generic definition.""" - - return audit_file.declaration_kind is DeclarationKind.AUDIT - - -def singular_audit_resources( - *, - references: tuple[CompileSqlReference, ...], - audit_file: DiscoveredAuditFile, - audit_name: str, -) -> tuple[ResourceIdentity, ...]: - """Return the resources a singular audit references, reporting P004 if not cross-resource.""" - - model_names: frozenset[str] = frozenset( - reference.ref_name for reference in references if reference.ref_kind == SqlReferenceKind.REF - ) - companion_kinds: dict[SqlReferenceKind, ResourceKind] = { - SqlReferenceKind.SOURCE: ResourceKind.SOURCE, - SqlReferenceKind.SEED: ResourceKind.SEED, - SqlReferenceKind.TABLE_FUNCTION: ResourceKind.FUNCTION, - } - companions: frozenset[ResourceIdentity] = frozenset( - ResourceIdentity(companion_kinds[SqlReferenceKind(reference.ref_kind)], reference.ref_name) - for reference in references - if reference.ref_kind in companion_kinds - ) - resources: tuple[ResourceIdentity, ...] = tuple( - sorted({*(ResourceIdentity(ResourceKind.MODEL, name) for name in model_names), *companions}) - ) - if len(model_names) < _CROSS_MODEL_MINIMUM and not (model_names and companions): - message, help_text = _single_resource_problem( - model_names=model_names, companions=companions - ) - report_compile_diagnostic( - key=( - SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE, - audit_file.relative_path.as_posix(), - audit_name, - ), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE, - message=f"Singular audit '{audit_name}' in {audit_file.relative_path} {message}", - resource_type=CompiledResourceType.AUDIT, - resource_name=audit_name, - path=audit_file.relative_path, - help=help_text, - ), - ) - return resources - - -def _single_resource_problem( - *, model_names: frozenset[str], companions: frozenset[ResourceIdentity] -) -> tuple[str, str]: - if len(model_names) == 1: - model_name: str = next(iter(model_names)) - return ( - f"checks only model '{model_name}'; singular audits must be cross-resource", - f"attach a generic audit to '{model_name}' instead: write it in an audits/generic/ " - "role and select FROM @relation, joining @relation to itself for self-join checks", - ) - if companions: - return ( - "checks only sources or seeds; singular audits must reference at least one model", - "attach YAML audits to the source or seed instead", - ) - return ( - "references no SQLBuild resource; singular audits must reference two or more models, or " - "a model plus a source, seed, or table function via " - f"{SqlReferenceKind.REF.placeholder_call()}, {SqlReferenceKind.SOURCE.placeholder_call()}, " - f"or {SqlReferenceKind.SEED.placeholder_call()}", - "reference resources through SQLBuild calls instead of hard-coded relation names, or " - "attach a generic audit to the resource being checked", - ) - - -def resolve_audit_severity( - *, - instance_severity: str | AuditSeverity | None, - default_severity: str | AuditSeverity | None, - audit_label: str, -) -> AuditSeverity: - """Resolve audit severity from instance, project default, or error fallback.""" - - valid_values: tuple[str, ...] = tuple(severity.value for severity in AuditSeverity) - if instance_severity is not None: - try: - return AuditSeverity(instance_severity) - except ValueError as error: - raise CompileInputError( - f"{audit_label}: unknown severity '{instance_severity}'; " - f"valid values: {', '.join(valid_values)}" - ) from error - if default_severity is not None: - try: - return AuditSeverity(default_severity) - except ValueError as error: - raise CompileInputError( - "settings.default_audit_severity in sqlbuild_project.toml: " - f"unknown value '{default_severity}'; valid values: {', '.join(valid_values)}" - ) from error - return AuditSeverity.ERROR - - -def measurement_policy_severity(thresholds: MeasurementThresholds) -> AuditSeverity: - """Return the effective scheduling severity for a measurement policy.""" - - return AuditSeverity.ERROR if thresholds.error is not None else AuditSeverity.WARN - - -def resolve_audit_run_scope( - *, - instance_run_scope: str | None, - default_run_scope: str | None, -) -> str: - """Resolve audit run scope from instance, project default, or delta/final fallback.""" - - from sqlbuild.compiler.auditing.types import AuditRunScope - - valid_values: frozenset[str] = frozenset(s.value for s in AuditRunScope) - if instance_run_scope is not None: - if instance_run_scope not in valid_values: - raise CompileInputError( - f"unknown audit run_scope '{instance_run_scope}'; " - f"valid values: {', '.join(sorted(valid_values))}" - ) - return instance_run_scope - if default_run_scope is not None: - if default_run_scope not in valid_values: - raise CompileInputError( - f"settings.default_audit_run_scope in sqlbuild_project.toml: " - f"unknown value '{default_run_scope}'; " - f"valid values: {', '.join(sorted(valid_values))}" - ) - return default_run_scope - return AuditRunScope.DELTA_AND_FINAL - - -def validate_model_attached_audit_references( - *, - references: tuple[CompileSqlReference, ...], - attached_target_kind: str, - attached_target_name: str, - audit_label: str, -) -> None: - """Validate that a model-attached generic audit references the attached model.""" - - if attached_target_kind != AttachedAuditTargetKind.MODEL: - return - ref_names: frozenset[str] = frozenset( - ref.ref_name for ref in references if ref.ref_kind == SqlReferenceKind.REF - ) - if attached_target_name not in ref_names: - raise CompileInputError( - f"{audit_label}: model-attached audit must reference the attached model " - f"'{attached_target_name}' via {SqlReferenceKind.REF.placeholder_call()}" - ) - - -def _str_from_dict(*, values: dict[str, object], key: str) -> str | None: - """Extract a string value from a dict.""" - - raw: object | None = values.get(key) - return raw if isinstance(raw, str) else None - - -def _bool_from_dict(*, values: dict[str, object], key: str) -> bool: - """Extract a bool value from a dict.""" - - raw: object | None = values.get(key) - return raw if isinstance(raw, bool) else False - - -def _argument_references( - *expansions: AuthoredSqlExpansionResult | None, -) -> tuple[CompileSqlReference, ...]: - references: list[CompileSqlReference] = [] - for expansion in expansions: - if expansion is not None: - references.extend(expansion.argument_references) - return tuple(references) - - -def _audit_reference_origin(audit_file: DiscoveredAuditFile) -> SqlReferenceOrigin: - return SqlReferenceOrigin( - file_path=audit_file.file_path, - relative_path=audit_file.relative_path, - contents=audit_file.contents, - ) - - -def _combined_references( - *sql_values: str | None, syntax: SqlLexicalSyntax, origin: SqlReferenceOrigin -) -> tuple[CompileSqlReference, ...]: - """Extract references from independently compiled measurement/evidence queries.""" - - references: list[CompileSqlReference] = [] - for sql in sql_values: - if sql is not None: - references.extend(extract_sql_references(sql=sql, syntax=syntax, origin=origin)) - return tuple(dict.fromkeys(references)) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index fd6285d374..f97ddcd479 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -2,29 +2,17 @@ from __future__ import annotations -import inspect -import re -from dataclasses import dataclass, field, fields, replace -from functools import partial -from inspect import Parameter, Signature +from dataclasses import dataclass, fields from pathlib import Path from typing import Any, cast -from sqlbuild.compiler.attachments.main._pair_native_seed_files import pair_native_seed_files -from sqlbuild.compiler.compile._helpers.analysis.validation import validate_sql_syntax +import sqlbuild._native as _native +from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text +from sqlbuild.compiler.compile._helpers.analysis.validation import sql_analysis_opt_out_help from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - _model_sql_validation_gate, - _resolve_model_schema, - build_layered_model_values, - build_model_header_schema_entry, - build_native_model_config, - find_schema_model_match, + legacy_schema_rows, native_model_config_session, - native_path_default, - strip_model_header_metadata_from_config, validate_declared_schema_models_are_attached, - validate_model_config, - validate_model_references, ) from sqlbuild.compiler.compile._helpers.attachment.references import ( build_known_function_names, @@ -33,100 +21,54 @@ build_known_source_names, build_known_table_function_names, ) -from sqlbuild.compiler.compile._helpers.diagnostics.sql_analysis_opt_outs import ( - rejected_sql_analysis_opt_out, -) -from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( - merge_call_site_references, - resource_references, -) -from sqlbuild.compiler.compile._helpers.named_declarations.core import ( - named_declaration_usages, -) -from sqlbuild.compiler.compile._helpers.named_declarations.model_schemas import ( - model_schema_enum_declarations, -) -from sqlbuild.compiler.compile._helpers.refs.cache import cached_sql_reference_extractor -from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references -from sqlbuild.compiler.compile._helpers.render.arguments import render_parameterized_sql -from sqlbuild.compiler.compile._helpers.render.context_templates import ( - build_model_context_values, -) +from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsics_analysis_sql, - get_validated_model_cursor_intrinsics, - reject_cursor_intrinsics, + cursor_intrinsic_semantics, ) from sqlbuild.compiler.compile._helpers.render.declarations import ( - build_model_declaration_indexes, - expand_scanned_declaration_references, - resolve_declaration_context, - resolve_enum_contract_columns, + render_constant_declaration, ) from sqlbuild.compiler.compile._helpers.render.macros import ( - expand_sql_macros_result, + macro_expansion_classes, ) -from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes from sqlbuild.compiler.compile._helpers.render.sql_vars import ( - applied_interpolation, - expand_authored_sql_result, interpolate_sql_batch, ) from sqlbuild.compiler.compile._helpers.render.templating import ( - contains_template_data, expand_effective_vars, expand_template_data, + record_template_reads, ) -from sqlbuild.compiler.compile.constants import ( - MACRO_CALL_PATTERN, -) +from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment +from sqlbuild.compiler.compile.constants import EXPLICIT_REFERENCE_KINDS from sqlbuild.compiler.compile.exceptions import CompileInputError, MacroArgumentError from sqlbuild.compiler.compile.models import ( - AuthoredSqlExpansionResult, - CompiledSqlExpansion, - CompileModelConfig, - CompileModelInput, - CompileSeedInput, - CompileSqlReference, - DeclarationExpansionContext, - DeclarationExpansionResult, DeclarationResolutionContext, - DeclarationScopeResolver, - ExpansionSpan, - HookExpansionResult, LoadedMacro, - MacroContext, - MacroExpansionResult, - MappedOffset, ModelInputBuildContext, ModelResourceNames, - ModelValidationRequest, ModelValidatorContext, NativeModelConfigInputs, - NativeModelConfigSession, - SqlAnalysisOptOutRequest, SqlInterpolation, - SqlReferenceOrigin, - SqlReferenceScan, - SqlReferenceSourceMap, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, SqlReferenceExtractor +from sqlbuild.compiler.compile.types import ModelInput +from sqlbuild.compiler.compiled_project.main._model_loop_classes import model_loop_classes +from sqlbuild.compiler.compiled_project.types import SpanRow from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME -from sqlbuild.compiler.discovery.main._model_query_start import get_model_query_start from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, - DiscoveredHookFunction, DiscoveredProjectInputs, DiscoveredSchemaFile, - DiscoveredSeedFile, DiscoveredSqlHookFile, DiscoveredSqlModelFile, EnumDeclaration, - ModelSchemaDeclaration, NamedSqlHookEntry, PythonHookEntry, - SqlHookEntry, ) +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge +from sqlbuild.compiler.model_config.main._native_contract_classes import native_contract_classes from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( scan_native_declaration_references, ) @@ -135,42 +77,20 @@ from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, - ResourceIdentity, - UsageRecord, VisibilityRecord, ) from sqlbuild.compiler.scopes.types import ( DeclarationKind, - ResourceKind, - ScopeKind, - UsageKind, - VisibilityReason, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.spec.contracts.models import ( - DefaultsConfig, LocalConfig, ProjectConfig, - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, SettingsConfig, - SourceLocation, TargetConfig, ) -_HOOK_TEMPLATE_PATTERN: re.Pattern[str] = re.compile(r"\$\{[^}]+\}") -_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) -_MODEL_HEADER_COLUMNS_KEY: str = "columns" -_MODEL_HEADER_DESCRIPTION_KEY: str = "description" -_REUSABLE_MODEL_HEADER_KEYS: frozenset[str] = frozenset( - {_MODEL_HEADER_COLUMNS_KEY, _MODEL_HEADER_DESCRIPTION_KEY} -) -_HOOK_CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset( - {"ctx", "context", "_ctx", "hook_context"} -) - @dataclass(frozen=True) class _VisibleModelDeclarations: @@ -187,50 +107,6 @@ class _VisibleModelDeclarations: inaccessible_macros: dict[str, DeclarationRecord] -@dataclass -class _VisibleModelDeclarationCache: - context: ModelInputBuildContext - reusable_by_parent: bool - resource_specific: frozenset[ResourceIdentity] = frozenset() - by_parent: dict[Path, _VisibleModelDeclarations] = field(default_factory=dict) - - @classmethod - def build(cls, context: ModelInputBuildContext) -> _VisibleModelDeclarationCache: - resolver: DeclarationScopeResolver | None = context.declaration_resolver - return cls( - context=context, - reusable_by_parent=resolver is not None, - resource_specific=( - frozenset( - declaration.identity.owner - for declaration in resolver.lookup.index.declarations - if declaration.scope is ScopeKind.PRIVATE - and declaration.identity.owner is not None - ) - | frozenset(resolver.lookup.grants_by_resource) - if resolver is not None - else frozenset() - ), - ) - - def for_model( - self, *, model_file: DiscoveredSqlModelFile, consumer: ResourceIdentity - ) -> _VisibleModelDeclarations: - parent: Path = model_file.file_path.parent - if consumer in self.resource_specific: - return _build_visible_declaration_indexes(model_file=model_file, context=self.context) - cached: _VisibleModelDeclarations | None = self.by_parent.get(parent) - if self.reusable_by_parent and cached is not None: - return _rebind_visible_declarations(declarations=cached, consumer=consumer) - resolved: _VisibleModelDeclarations = _build_visible_declaration_indexes( - model_file=model_file, - context=self.context, - ) - if self.reusable_by_parent: - self.by_parent[parent] = resolved - return resolved - - @dataclass(frozen=True) class _ModelValidationContext: effective_settings: SettingsConfig @@ -238,7 +114,7 @@ class _ModelValidationContext: project_config_path: Path defer_model_sql_validation: bool external_sql_reference_resolver: ExternalSqlReferenceResolver | None - extract_references: SqlReferenceExtractor + sql_lexical_syntax: SqlLexicalSyntax known_model_names: set[str] known_seed_names: set[str] known_source_names: set[str] @@ -255,161 +131,23 @@ class _ModelInputLoop: validation_context: _ModelValidationContext sql_hook_definitions: dict[str, DiscoveredSqlHookFile] legacy_schema_files: tuple[DiscoveredSchemaFile, ...] - reusable_config_cache: _ReusableModelConfigCache - declaration_cache: _VisibleModelDeclarationCache native_declaration_references: dict[Path, NativeDeclarationScan] -@dataclass(frozen=True) -class _HookExpansionContext: - file_path: Path - effective_vars: dict[str, object] - context_values: dict[str, str | None] - loaded_macros: dict[str, LoadedMacro] - macro_context: MacroContext - declaration_expansion: DeclarationExpansionContext - sql_hook_definitions: dict[str, DiscoveredSqlHookFile] - consumer: ResourceIdentity | DeclarationIdentity - facts: _HookExpansionFacts - sql_lexical_syntax: SqlLexicalSyntax - reference_origin: SqlReferenceOrigin | None = None - - @dataclass -class _HookExpansionFacts: - usages: list[UsageRecord] - references: list[CompileSqlReference] = field(default_factory=list) +class _ModelLoopHost: + """The attachment steps the native model loop calls back into, one model index at a time.""" - def add(self, usages: tuple[UsageRecord, ...]) -> None: - self.usages.extend(usages) + loop: _ModelInputLoop + interpolations: dict[Path, SqlInterpolation] - def add_references(self, references: tuple[CompileSqlReference, ...]) -> None: - self.references.extend(references) - - def reads_since(self, start: int) -> tuple[SqlResourceRef, ...]: - return resource_references(tuple(self.references[start:])) - - -@dataclass -class _ReusableModelConfigCache: - defaults: DefaultsConfig - path_defaults: dict[str, dict[str, object]] - target_config: TargetConfig | None - reusable_by_path_default: dict[str | None, bool] = field(default_factory=dict) - inherited_values_by_path_default: dict[str | None, dict[str, object]] = field( - default_factory=dict - ) - reusable_metadata_by_identity: dict[int, tuple[object, bool]] = field(default_factory=dict) - configs: dict[str | None, CompileModelConfig] = field(default_factory=dict) - - def get( - self, - *, - matched_path_default: str | None, - model_header_values: dict[str, object], - ) -> CompileModelConfig | None: - reusable_metadata: dict[str, object] | None = self._reusable_metadata(model_header_values) - if reusable_metadata is None or not self._is_reusable(matched_path_default): - return None - cached: CompileModelConfig | None = self.configs.get(matched_path_default) - if cached is None: - return None - values: dict[str, object] = dict(cached.values) - values.update(reusable_metadata) - return replace(cached, values=values) - - def remember( - self, - *, - matched_path_default: str | None, - model_header_values: dict[str, object], - config: CompileModelConfig, - ) -> None: - reusable_metadata: dict[str, object] | None = self._reusable_metadata(model_header_values) - if reusable_metadata is None or not self._is_reusable(matched_path_default): - return - inherited_values: dict[str, object] = self.inherited_values_by_path_default[ - matched_path_default - ] - reusable_values: dict[str, object] = { - key: value for key, value in config.values.items() if key not in reusable_metadata - } - reusable_values.update( - (key, inherited_values[key]) for key in reusable_metadata if key in inherited_values + def constant_text(self, *, declaration: ConstantDeclaration, file_path: Path) -> str: + return render_constant_declaration( + declaration=declaration, + value_renderer=self.loop.context.value_renderer, + collection_rendering=self.loop.context.collection_rendering, + file_path=file_path, ) - self.configs[matched_path_default] = replace(config, values=reusable_values) - - def _reusable_metadata( - self, - model_header_values: dict[str, object], - ) -> dict[str, object] | None: - if not model_header_values: - return {} - if not model_header_values.keys() <= _REUSABLE_MODEL_HEADER_KEYS: - return None - if _contains_dynamic_or_unsupported_reusable_metadata( - model_header_values.get(_MODEL_HEADER_DESCRIPTION_KEY) - ): - return None - if _MODEL_HEADER_COLUMNS_KEY not in model_header_values: - return model_header_values - metadata: object = model_header_values[_MODEL_HEADER_COLUMNS_KEY] - cached: tuple[object, bool] | None = self.reusable_metadata_by_identity.get(id(metadata)) - if cached is not None and cached[0] is metadata: - return model_header_values if cached[1] else None - reusable: bool = not _contains_dynamic_or_unsupported_reusable_metadata(metadata) - self.reusable_metadata_by_identity[id(metadata)] = (metadata, reusable) - if not reusable: - return None - return model_header_values - - def _is_reusable(self, matched_path_default: str | None) -> bool: - cached: bool | None = self.reusable_by_path_default.get(matched_path_default) - if cached is not None: - return cached - layered_values: dict[str, object] = build_layered_model_values( - defaults=self.defaults, - path_defaults=self.path_defaults, - matched_path_default=matched_path_default, - model_header_values={}, - ) - target_namespace_values: tuple[object, object] = ( - None if self.target_config is None else self.target_config.database, - None if self.target_config is None else self.target_config.schema, - ) - reusable: bool = not contains_template_data( - (layered_values, target_namespace_values) - ) and not _contains_mutable_nested_config(layered_values) - self.reusable_by_path_default[matched_path_default] = reusable - self.inherited_values_by_path_default[matched_path_default] = layered_values - return reusable - - -def _contains_mutable_nested_config(values: dict[str, object]) -> bool: - if any(key in values for key in _MODEL_HOOK_KEYS): - return True - - def is_mutable(value: object) -> bool: - if isinstance(value, dict | list | set): - return True - if isinstance(value, tuple): - return any(is_mutable(item) for item in value) - return value is not None and not isinstance(value, str | int | float | bool) - - return any(is_mutable(value) for value in values.values()) - - -def _contains_dynamic_or_unsupported_reusable_metadata(value: object) -> bool: - if isinstance(value, str): - return contains_template_data(value) or MACRO_CALL_PATTERN.search(value) is not None - if isinstance(value, dict): - return any( - not isinstance(key, str) or _contains_dynamic_or_unsupported_reusable_metadata(item) - for key, item in value.items() - ) - if isinstance(value, list | tuple): - return any(_contains_dynamic_or_unsupported_reusable_metadata(item) for item in value) - return value is not None and not isinstance(value, int | float | bool) def build_model_inputs( @@ -419,25 +157,22 @@ def build_model_inputs( no_sql_validation: bool = False, defer_model_sql_validation: bool = False, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - reference_cache_dir: Path | None = None, -) -> tuple[CompileModelInput, ...]: + native_project: _native.NativeCompiledProject, +) -> tuple[ModelInput, ...]: """Attach schema metadata to discovered model files.""" legacy_schema_files: tuple[DiscoveredSchemaFile, ...] = tuple( schema_file for schema_file in discovered_inputs.schema_files if schema_file.model_entries ) - with cached_sql_reference_extractor( - root=reference_cache_dir, syntax=context.sql_lexical_syntax - ) as extract_references: - return _build_model_inputs( - discovered_inputs=discovered_inputs, - context=context, - no_sql_validation=no_sql_validation, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - extract_references=extract_references, - legacy_schema_files=legacy_schema_files, - ) + return _build_model_inputs( + discovered_inputs=discovered_inputs, + context=context, + no_sql_validation=no_sql_validation, + defer_model_sql_validation=defer_model_sql_validation, + external_sql_reference_resolver=external_sql_reference_resolver, + legacy_schema_files=legacy_schema_files, + native_project=native_project, + ) def _build_model_inputs( @@ -447,9 +182,9 @@ def _build_model_inputs( no_sql_validation: bool, defer_model_sql_validation: bool, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - extract_references: SqlReferenceExtractor, legacy_schema_files: tuple[DiscoveredSchemaFile, ...], -) -> tuple[CompileModelInput, ...]: + native_project: _native.NativeCompiledProject, +) -> tuple[ModelInput, ...]: effective_vars: dict[str, object] = context.effective_vars effective_settings: SettingsConfig = context.effective_settings known_model_names: set[str] = build_known_ref_names(discovered_inputs) @@ -476,7 +211,7 @@ def _build_model_inputs( project_config_path=(discovered_inputs.project_dir or Path()) / PROJECT_CONFIG_FILENAME, defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, - extract_references=extract_references, + sql_lexical_syntax=context.sql_lexical_syntax, known_model_names=known_model_names, known_seed_names=known_seed_names, known_source_names=known_source_names, @@ -527,12 +262,6 @@ def _build_model_inputs( validation_context=validation_context, sql_hook_definitions=sql_hook_definitions, legacy_schema_files=legacy_schema_files, - reusable_config_cache=_ReusableModelConfigCache( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - target_config=context.target_config, - ), - declaration_cache=_VisibleModelDeclarationCache.build(context), native_declaration_references=dict( zip( (model_file.file_path for model_file in prepared_files), @@ -545,17 +274,9 @@ def _build_model_inputs( ) ), ) - model_inputs: list[CompileModelInput] = [] - model_file: DiscoveredSqlModelFile - for model_file in discovered_inputs.model_files: - model_inputs.append( - _build_model_input( - loop=loop, - model_file=model_file, - interpolation=interpolations[model_file.file_path], - ) - ) - + model_inputs: list[ModelInput] = _run_native_model_loop( + loop=loop, interpolations=interpolations, native_project=native_project + ) validate_declared_schema_models_are_attached( model_inputs=tuple(model_inputs), schema_files=discovered_inputs.schema_files, @@ -563,568 +284,133 @@ def _build_model_inputs( return tuple(model_inputs) -def _build_model_input( +def _run_native_model_loop( *, loop: _ModelInputLoop, - model_file: DiscoveredSqlModelFile, - interpolation: SqlInterpolation, -) -> CompileModelInput: - discovered_inputs: DiscoveredProjectInputs = loop.discovered_inputs - context: ModelInputBuildContext = loop.context - validation_context: _ModelValidationContext = loop.validation_context - sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = loop.sql_hook_definitions - legacy_schema_files: tuple[DiscoveredSchemaFile, ...] = loop.legacy_schema_files - reusable_config_cache: _ReusableModelConfigCache = loop.reusable_config_cache - declaration_cache: _VisibleModelDeclarationCache = loop.declaration_cache - effective_vars: dict[str, object] = context.effective_vars - effective_target_name: str | None = context.effective_target_name - run_id: str = context.run_id - macro_context: MacroContext = context.macro_context - loaded_macros: dict[str, LoadedMacro] = context.loaded_macros - model_identity: ResourceIdentity = ResourceIdentity( - ResourceKind.MODEL, model_file.file_path.stem - ) - declarations: _VisibleModelDeclarations = declaration_cache.for_model( - model_file=model_file, consumer=model_identity - ) - native_config: NativeModelConfigSession = validation_context.validators.native_config - matched_path_default: str | None = native_path_default( - session=native_config, model_file=model_file - ) - effective_config: CompileModelConfig | None = reusable_config_cache.get( - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, - ) - if effective_config is None: - effective_config = build_native_model_config( - session=native_config, - model_file=model_file, - matched_path_default=matched_path_default, - ) - reusable_config_cache.remember( - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, - config=effective_config, - ) - model_schema: ModelSchemaDeclaration | None = _resolve_model_schema( - values=effective_config.values, - model_name=model_file.file_path.stem, - public_model_schemas=context.public_model_schemas, - ) - model_schema_columns: tuple[SchemaColumn, ...] | None = ( - model_schema.columns if model_schema is not None else None - ) - validate_python_hook_config( - values=effective_config.values, - model_name=model_file.file_path.stem, - hook_functions=discovered_inputs.hook_functions, - provider_names=frozenset(provider.name for provider in discovered_inputs.providers), - ) - named_usages: tuple[UsageRecord, ...] = ( - *( - named_declaration_usages( - resolver=context.declaration_resolver, - kind=DeclarationKind.SCHEMA, - name=model_schema.name, - consumer=model_identity, - consumer_path=model_file.relative_path, - ) - if model_schema is not None - else () - ), - *_python_hook_usages( - values=effective_config.values, - resolver=context.declaration_resolver, - consumer=model_identity, - consumer_path=model_file.relative_path, - ), - ) - var_substituted_sql: str = applied_interpolation(interpolation) - declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, - enum_visibility=declarations.enum_visibility, - constant_visibility=declarations.constant_visibility, - macros=declarations.macros, - macro_records=declarations.macro_records, - inaccessible_macros=declarations.inaccessible_macros, - consumer=model_identity, - ) - declaration_expansion: DeclarationExpansionResult = expand_scanned_declaration_references( - sql=var_substituted_sql, - references=loop.native_declaration_references[model_file.file_path], - file_path=model_file.file_path, - declarations=declaration_context, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - ) - declaration_expanded_sql: str = declaration_expansion.sql - try: - macro_expansion: MacroExpansionResult = expand_sql_macros_result( - sql=declaration_expanded_sql, - file_path=model_file.file_path, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_resolver=context.declaration_resolver, - declarations=( - declaration_context if context.declaration_resolver is not None else None + interpolations: dict[Path, SqlInterpolation], + native_project: _native.NativeCompiledProject, +) -> list[ModelInput]: + """Run the model loop natively; models it cannot finish yet call back into the attachment.""" + + files: tuple[DiscoveredSqlModelFile, ...] = loop.discovered_inputs.model_files + settings: SettingsConfig = loop.validation_context.effective_settings + native_loop: _native.NativeModelLoop = _native.NativeModelLoop( + ( + loop.validation_context.validators.native_config.validator, + native_contract_classes(), + model_loop_classes( + audit_factories=loop.discovered_inputs.audit_factories, + context=loop.context, + external_resolver=loop.validation_context.external_sql_reference_resolver, + host_inputs={ + "report_diagnostic": report_compile_diagnostic, + "record_template_reads": record_template_reads, + "visible_declarations": _VisibleModelDeclarations, + "config_session": loop.validation_context.validators.native_config, + "syntax_help": sql_analysis_opt_out_help(model_owned=True), + "source_syntax_help": sql_analysis_opt_out_help(model_owned=False), + "macros": { + "loaded_macros": loop.context.loaded_macros, + "macro_context": loop.context.macro_context, + "bridge": active_macro_bridge(), + "scoped": loop.context.declaration_resolver is not None, + "declaration_context": DeclarationResolutionContext, + "macro_argument_error": MacroArgumentError, + "expansion": macro_expansion_classes(), + "generic_syntax": SqlLexicalSyntax().native_mapping, + }, + "sql_hooks": { + "named_sql_hook_entry": NamedSqlHookEntry, + "python_hook_entry": PythonHookEntry, + "definitions": loop.sql_hook_definitions, + "effective_vars": loop.context.effective_vars, + "environment": UnicodeEnvironment(), + "render_project_var": render_project_var_text, + "sql_resource_ref": SqlResourceRef, + "explicit_kinds": EXPLICIT_REFERENCE_KINDS, + "declaration_identity": DeclarationIdentity, + "sql_hook_kind": DeclarationKind.SQL_HOOK, + }, + "hook_functions": loop.discovered_inputs.hook_functions, + "provider_names": frozenset( + provider.name for provider in loop.discovered_inputs.providers + ), + }, ), - consumer=model_identity, - ) - except MacroArgumentError as error: - location: SourceLocation | None = ( - _authored_model_location( - model_file=model_file, - sql=declaration_expanded_sql, - offset=error.offset, - passes=(declaration_expansion.spans,), - ) - if var_substituted_sql == model_file.query_sql - else None - ) - if location is None: - raise - raise error.located(location) from None - expanded_query_sql: str = macro_expansion.sql - expanded_query_sql = get_validated_model_cursor_intrinsics( - sql=expanded_query_sql, - config_values=effective_config.values, - model_name=model_file.file_path.stem, - ) - raw_placeholders: object | None = effective_config.values.get("placeholders") - sql_validation_placeholders: dict[str, str] | None = ( - {str(k): str(v) for k, v in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else None - ) - sql_validation_enabled, references, rejected_opt_out = _validate_model_input( - context=validation_context, - model_file=model_file, - config=effective_config, - expanded_query_sql=expanded_query_sql, - sql_validation_placeholders=sql_validation_placeholders, - model_schema_columns=model_schema_columns, - argument_references=macro_expansion.argument_references, - reference_source_map=( - SqlReferenceSourceMap( - body_start=partial( - get_model_query_start, - contents=model_file.contents, - query_sql=model_file.query_sql, - ), - passes=(declaration_expansion.spans, macro_expansion.spans), - ) - if var_substituted_sql == model_file.query_sql - and expanded_query_sql == macro_expansion.sql - else None - ), - ) - hook_expansion: HookExpansionResult = expand_model_hook_macros_result( - values=effective_config.values, - model_file=_model_reference_origin(model_file=model_file), - effective_vars=effective_vars, - context_values=build_model_context_values( - values=effective_config.values, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - include_target_values=True, ), - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_expansion=DeclarationExpansionContext( - declarations=DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, + loop.context.sql_lexical_syntax.native_mapping, + cursor_intrinsic_semantics(), + ( + ( + settings.sql_analysis, + loop.validation_context.no_sql_validation, + loop.validation_context.defer_model_sql_validation, + settings.require_sql_analysis, ), - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - resolver=context.declaration_resolver, - ), - sql_lexical_syntax=context.sql_lexical_syntax, - sql_hook_definitions=sql_hook_definitions, - consumer=model_identity, - ) - expanded_config: CompileModelConfig = _hook_expanded_config( - config=effective_config, - hook_expansion=hook_expansion, - model_file=model_file, - validation_context=validation_context, - ) - hook_name: str - for hook_name in ("pre_hooks", "post_hooks"): - hook_value: object | None = expanded_config.values.get(hook_name) - if isinstance(hook_value, list | tuple): - hook_entry: object - for hook_entry in hook_value: - if isinstance(hook_entry, SqlHookEntry): - reject_cursor_intrinsics( - sql=hook_entry.statement, - context=f"Model '{model_file.file_path.stem}' {hook_name}", - ) - header_schema_entry: SchemaModelEntry | None = build_model_header_schema_entry( - model_name=model_file.file_path.stem, - model_header_values=expanded_config.values, - file_path=model_file.relative_path, - column_locations=model_file.header_column_locations, - model_schema_columns=model_schema_columns, - model_schema_name=model_schema.name if model_schema is not None else None, - model_schema_description=model_schema.description if model_schema is not None else None, - audit_factories=discovered_inputs.audit_factories, - ) - model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) - header_schema_entry, enum_columns = resolve_enum_contract_columns( - schema_entry=header_schema_entry, - config_values=model_config.values, - enums=( - declarations.enums - if model_schema is None - else { - **declarations.enums, - **model_schema_enum_declarations( - schema=model_schema, resolver=context.declaration_resolver - ), - } - ), - ) - generated_usages: tuple[UsageRecord, ...] = _generated_enum_usages( - enum_columns=enum_columns, - declarations=declarations, - consumer=model_identity, - ) - model_declaration_usages: tuple[UsageRecord, ...] = tuple( - dict.fromkeys( ( - *declaration_expansion.usages, - *generated_usages, - *hook_expansion.usages, - *named_usages, - ) - ) - ) - _reject_legacy_schema_match(model_file=model_file, schema_files=legacy_schema_files) - if header_schema_entry is None: - return CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), - ) - if var_substituted_sql == model_file.query_sql - else None + str(loop.validation_context.project_config_path), + legacy_schema_rows(loop.legacy_schema_files), ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, - ) - - return CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), - ) - if var_substituted_sql == model_file.query_sql - else None ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - schema_entry=header_schema_entry, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, - ) - - -def _hook_expanded_config( - *, - config: CompileModelConfig, - hook_expansion: HookExpansionResult, - model_file: DiscoveredSqlModelFile, - validation_context: _ModelValidationContext, -) -> CompileModelConfig: - validate_model_references( - context=validation_context.validators, - model_file=model_file, - references=hook_expansion.references, - ) - return CompileModelConfig( - values=hook_expansion.values, - model_header_keys=config.model_header_keys, - matched_path_default=config.matched_path_default, - logical_schema=config.logical_schema, - layer_schema=config.layer_schema, - logical_database=config.logical_database, - time_travel_retention=config.time_travel_retention, - table_type=config.table_type, - ) - - -def _model_reference_origin( - *, model_file: DiscoveredSqlModelFile, source_map: SqlReferenceSourceMap | None = None -) -> SqlReferenceOrigin: - return SqlReferenceOrigin( - file_path=model_file.file_path, - relative_path=model_file.relative_path, - contents=model_file.contents, - resource_type=CompiledResourceType.MODEL, - resource_name=model_file.file_path.stem, - source_map=source_map, - ) - - -def _validate_model_input( - *, - context: _ModelValidationContext, - model_file: DiscoveredSqlModelFile, - config: CompileModelConfig, - expanded_query_sql: str, - sql_validation_placeholders: dict[str, str] | None, - model_schema_columns: tuple[SchemaColumn, ...] | None, - argument_references: tuple[CompileSqlReference, ...], - reference_source_map: SqlReferenceSourceMap | None, -) -> tuple[bool, tuple[CompileSqlReference, ...], SourceLocation | None]: - model_name: str = model_file.file_path.stem - sql_validation_enabled: bool = _model_sql_validation_gate( - effective_settings=context.effective_settings, - no_sql_validation=context.no_sql_validation, - model_config=config, - ) - rejected_opt_out: SourceLocation | None = ( - None - if sql_validation_enabled - else rejected_sql_analysis_opt_out( - SqlAnalysisOptOutRequest( - model_file=model_file, - config=config, - settings=context.effective_settings, - no_sql_validation=context.no_sql_validation, - query_sql=cursor_intrinsics_analysis_sql( - sql=expanded_query_sql, cursor_type=config.values.get("cursor_type") - ), - placeholders=sql_validation_placeholders, - project_config_path=context.project_config_path, - ) - ) ) - sql_validation_enabled = sql_validation_enabled or rejected_opt_out is not None - if sql_validation_enabled and not context.defer_model_sql_validation: - validate_sql_syntax( - query_sql=cursor_intrinsics_analysis_sql( - sql=expanded_query_sql, - cursor_type=config.values.get("cursor_type"), - ), - model_name=model_name, - file_path=model_file.file_path, - placeholders=sql_validation_placeholders, + host: _ModelLoopHost = _ModelLoopHost(loop=loop, interpolations=interpolations) + inputs, (finished_natively, config_builds, declaration_contexts, hook_statements, described) = ( + native_loop.run( + native_project, + host, + [ + ( + ( + model_file.file_path.stem, + model_file.relative_path.as_posix(), + str(model_file.relative_path), + str(model_file.file_path), + ), + ( + model_file.header_column_locations, + model_file.header_values, + model_file.enum_declarations, + model_file.constant_declarations, + ), + model_file.query_sql, + model_file.contents, + loop.native_declaration_references.get(model_file.file_path), + _interpolation_row(interpolations[model_file.file_path]), + model_file, + ) + for model_file in files + ], ) - reference_scan: SqlReferenceScan = context.extract_references( - sql=expanded_query_sql, - origin=_model_reference_origin(model_file=model_file, source_map=reference_source_map), - ) - references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=reference_scan.references, - argument_references=argument_references, ) - validate_model_config( - context=context.validators, - request=ModelValidationRequest( - model_file=model_file, - config=config, - references=references, - declared_columns=model_schema_columns, - query_sql=expanded_query_sql, - ), + report_native_answer(stage=NativeStage.MODEL_CONFIG, kind="config_builds", units=config_builds) + report_native_answer( + stage=NativeStage.MODEL_LOOP, kind="declaration_contexts", units=declaration_contexts ) - return ( - sql_validation_enabled and not reference_scan.invalid_calls, - references, - rejected_opt_out, + report_native_answer( + stage=NativeStage.MODEL_LOOP, kind="native_model_inputs", units=finished_natively ) - - -def _build_visible_declaration_indexes( - *, model_file: DiscoveredSqlModelFile, context: ModelInputBuildContext -) -> _VisibleModelDeclarations: - local_enums: dict[str, EnumDeclaration] - local_constants: dict[str, ConstantDeclaration] - model_identity: ResourceIdentity = ResourceIdentity( - ResourceKind.MODEL, model_file.file_path.stem - ) - local_enums, local_constants = build_model_declaration_indexes(model_file=model_file) - visible: DeclarationResolutionContext = DeclarationResolutionContext( - enums=context.public_enums, - constants=context.public_constants, - ) - if ( - context.declaration_resolver is not None - and context.declaration_resolver.lookup.index.declarations + for stage, kind, units in ( + (NativeStage.MODEL_LOOP, "declaration_expansions", finished_natively), + (NativeStage.MODEL_LOOP, "cursor_intrinsic_validations", finished_natively), + (NativeStage.MODEL_CONFIG, "validations", finished_natively), + (NativeStage.MODEL_CONFIG, "header_metadata_parses", described), + (NativeStage.ATTACHMENTS, "cursor_intrinsic_rejections", hook_statements), ): - visible = resolve_declaration_context( - resolver=context.declaration_resolver, file_path=model_file.file_path - ) - return _VisibleModelDeclarations( - local_enums=local_enums, - local_constants=local_constants, - enums=visible.enums | local_enums, - constants=visible.constants | local_constants, - inaccessible_enums=visible.inaccessible_enums, - inaccessible_constants=visible.inaccessible_constants, - enum_visibility=visible.enum_visibility - | { - name: ( - VisibilityRecord( - model_identity, - DeclarationIdentity(DeclarationKind.ENUM, name, model_identity), - reason=VisibilityReason.PRIVATE_OWNER, - ), - ) - for name in local_enums - }, - constant_visibility=visible.constant_visibility - | { - name: ( - VisibilityRecord( - model_identity, - DeclarationIdentity(DeclarationKind.CONSTANT, name, model_identity), - reason=VisibilityReason.PRIVATE_OWNER, - ), - ) - for name in local_constants - }, - macros=visible.macros, - macro_records=visible.macro_records, - inaccessible_macros=visible.inaccessible_macros, - ) - - -def _rebind_visible_declarations( - *, declarations: _VisibleModelDeclarations, consumer: ResourceIdentity -) -> _VisibleModelDeclarations: - if not declarations.enum_visibility and not declarations.constant_visibility: - return declarations - return replace( - declarations, - enum_visibility=_rebind_visibility( - visibility=declarations.enum_visibility, consumer=consumer - ), - constant_visibility=_rebind_visibility( - visibility=declarations.constant_visibility, consumer=consumer - ), - ) - - -def _rebind_visibility( - *, - visibility: dict[str, tuple[VisibilityRecord, ...]], - consumer: ResourceIdentity, -) -> dict[str, tuple[VisibilityRecord, ...]]: - rebound: dict[str, tuple[VisibilityRecord, ...]] = {} - for name, records in visibility.items(): - rebound[name] = tuple(replace(record, resource=consumer) for record in records) - return rebound + report_native_answer(stage=stage, kind=kind, units=units) + return list(inputs) -def _generated_enum_usages( - *, - enum_columns: dict[str, EnumDeclaration], - declarations: _VisibleModelDeclarations, - consumer: ResourceIdentity, -) -> tuple[UsageRecord, ...]: - usages: list[UsageRecord] = [] - for enum in enum_columns.values(): - for visibility in declarations.enum_visibility.get(enum.name, ()): - usages.append( - UsageRecord( - consumer=consumer, - declaration=visibility.declaration, - kind=UsageKind.GENERATED, - through=visibility.through, - ) - ) - return tuple(usages) - - -def _reject_legacy_schema_match( - *, model_file: DiscoveredSqlModelFile, schema_files: tuple[DiscoveredSchemaFile, ...] -) -> None: - schema_match: tuple[SchemaModelEntry, DiscoveredSchemaFile] | None = find_schema_model_match( - model_file=model_file, - schema_files=schema_files, - ) - if schema_match is None: - return - schema_entry, schema_file = schema_match - raise CompileInputError( - f"Schema file {schema_file.relative_path} declares model '{schema_entry.name}', " - f"but model metadata must live in {model_file.relative_path} MODEL(...). " - "Move description, columns, and audits into the model header." - ) - - -def build_seed_inputs(discovered_inputs: DiscoveredProjectInputs) -> tuple[CompileSeedInput, ...]: - """Attach seed declarations to discovered seed CSV files.""" - - seed_declarations: list[tuple[SchemaSeedEntry, DiscoveredSchemaFile]] = [] - schema_file: DiscoveredSchemaFile - for schema_file in discovered_inputs.schema_files: - seed_entry: SchemaSeedEntry - for seed_entry in schema_file.seed_entries: - seed_declarations.append((seed_entry, schema_file)) - - return _native_seed_inputs( - seed_declarations=seed_declarations, seed_files=discovered_inputs.seed_files - ) - - -def _native_seed_inputs( - *, - seed_declarations: list[tuple[SchemaSeedEntry, DiscoveredSchemaFile]], - seed_files: tuple[DiscoveredSeedFile, ...], -) -> tuple[CompileSeedInput, ...]: - pairs: list[int] | None - missing: int | None - pairs, missing = pair_native_seed_files( - declaration_names=[seed_entry.name for seed_entry, _ in seed_declarations], - file_stems=[seed_file.file_path.stem for seed_file in seed_files], - ) - if pairs is None: - seed_entry, seed_schema_file = seed_declarations[cast(int, missing)] - raise CompileInputError( - f"Seed declaration '{seed_entry.name}' in {seed_schema_file.relative_path} " - "has no matching CSV file under seeds/", - ) - return tuple( - CompileSeedInput( - seed_file=seed_files[file_index], - schema_entry=seed_entry, - schema_file=seed_schema_file, - ) - for (seed_entry, seed_schema_file), file_index in zip(seed_declarations, pairs, strict=True) +def _interpolation_row( + interpolation: SqlInterpolation, +) -> tuple[str, list[tuple[str, str]], str | None, list[SpanRow]]: + return ( + interpolation.sql, + list(interpolation.reads), + interpolation.error, + [ + (span.source_start, span.source_end, span.output_start, span.output_end) + for span in interpolation.spans + ], ) @@ -1137,23 +423,11 @@ def build_effective_connection( ) -> dict[str, object]: """Merge base project connection with the selected environment overrides.""" - connection: dict[str, object] = dict(project_config.connection) - if target_config is not None: - connection_name: str | None = target_config.connection_name - if connection_name is not None: - if ( - connection_name not in project_config.connections - and connection_name not in local_config.connections - ): - raise CompileInputError( - f"Unknown connection '{connection_name}' selected by target. Define " - f"[connections.{connection_name}] in sqlbuild_project.toml or " - "sqlbuild_local.toml." - ) - connection.update(project_config.connections.get(connection_name, {})) - connection.update(local_config.connections.get(connection_name, {})) - connection.update(target_config.connection) - connection.update(local_config.connection) + connection, error = _native.effective_config_connection( + project_config, local_config, target_config + ) + if connection is None: + raise CompileInputError(error or "") return cast( dict[str, object], expand_template_data( @@ -1173,12 +447,11 @@ def build_effective_settings( ) -> SettingsConfig: """Merge project settings with local developer overrides.""" - values: dict[str, object] = { - field.name: getattr(project_config.settings, field.name) for field in fields(SettingsConfig) - } - setting_name: str - for setting_name in local_config.setting_overrides: - values[setting_name] = getattr(local_config.settings, setting_name) + values: dict[str, object] = _native.effective_config_settings( + (project_config.settings, local_config.settings), + local_config.setting_overrides, + [field.name for field in fields(SettingsConfig)], + ) return SettingsConfig(**cast(dict[str, Any], values)) @@ -1191,418 +464,9 @@ def build_effective_vars( ) -> dict[str, object]: """Merge effective vars using the locked precedence order.""" - values: dict[str, object] = dict(project_config.vars) - if target_config is not None: - values.update(target_config.vars) - values.update(local_config.vars) - values.update(cli_vars) - return expand_effective_vars(values) - - -def expand_model_hook_macros_result( - *, - values: dict[str, object], - model_file: SqlReferenceOrigin, - effective_vars: dict[str, object], - context_values: dict[str, str | None], - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - declaration_expansion: DeclarationExpansionContext, - sql_lexical_syntax: SqlLexicalSyntax, - sql_hook_definitions: dict[str, DiscoveredSqlHookFile] | None = None, - consumer: ResourceIdentity | None = None, -) -> HookExpansionResult: - """Expand hooks and retain declaration usages from inline and named SQL.""" - - expanded_values: dict[str, object] = dict(values) - facts: _HookExpansionFacts = _HookExpansionFacts(usages=[]) - hook_key: str - for hook_key in _MODEL_HOOK_KEYS: - raw_hook_value: object | None = expanded_values.get(hook_key) - if raw_hook_value is None: - continue - expanded_values[hook_key] = expand_sql_macros_in_value( - value=raw_hook_value, - context=_HookExpansionContext( - file_path=model_file.file_path, - effective_vars=effective_vars, - context_values=context_values, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_expansion=declaration_expansion, - sql_hook_definitions=sql_hook_definitions or {}, - consumer=consumer - or ResourceIdentity(ResourceKind.MODEL, model_file.file_path.stem), - facts=facts, - sql_lexical_syntax=sql_lexical_syntax, - reference_origin=model_file, - ), - hook_key=hook_key, - ) - return HookExpansionResult( - values=expanded_values, - usages=tuple(dict.fromkeys(facts.usages)), - references=tuple(dict.fromkeys(facts.references)), - ) - - -def validate_python_hook_config( - *, - values: dict[str, object], - model_name: str, - hook_functions: tuple[DiscoveredHookFunction, ...], - provider_names: frozenset[str] = frozenset(), -) -> None: - """Validate Python lifecycle hook references and explicit kwargs.""" - - hooks_by_name: dict[str, DiscoveredHookFunction] = { - hook_function.name: hook_function for hook_function in hook_functions - } - hook_key: str - for hook_key in sorted(_MODEL_HOOK_KEYS): - raw_value: object | None = values.get(hook_key) - if not isinstance(raw_value, list | tuple): - continue - hook_index: int - hook_entry: object - for hook_index, hook_entry in enumerate(raw_value): - if not isinstance(hook_entry, PythonHookEntry): - continue - hook_function: DiscoveredHookFunction | None = hooks_by_name.get(hook_entry.name) - if hook_function is None: - known_hook_names: str = ", ".join(sorted(hooks_by_name)) or "none discovered" - raise CompileInputError( - f"model '{model_name}' {hook_key}[{hook_index}] python(\"{hook_entry.name}\") " - f"references an unknown hook. Discovered hooks: {known_hook_names}" - ) - validate_python_hook_signature( - hook_entry=hook_entry, - hook_function=hook_function, - model_name=model_name, - hook_key=hook_key, - hook_index=hook_index, - provider_names=provider_names, - ) - - -def _python_hook_usages( - *, - values: dict[str, object], - resolver: DeclarationScopeResolver | None, - consumer: ResourceIdentity, - consumer_path: Path, -) -> tuple[UsageRecord, ...]: - usages: list[UsageRecord] = [] - hook_key: str - for hook_key in sorted(_MODEL_HOOK_KEYS): - raw_value: object | None = values.get(hook_key) - if not isinstance(raw_value, list | tuple): - continue - hook_entry: object - for hook_entry in raw_value: - if isinstance(hook_entry, PythonHookEntry): - usages.extend( - named_declaration_usages( - resolver=resolver, - kind=DeclarationKind.PYTHON_HOOK, - name=hook_entry.name, - consumer=consumer, - consumer_path=consumer_path, - ) - ) - return tuple(usages) - - -def validate_python_hook_signature( - *, - hook_entry: PythonHookEntry, - hook_function: DiscoveredHookFunction, - model_name: str, - hook_key: str, - hook_index: int, - provider_names: frozenset[str] = frozenset(), -) -> None: - signature: Signature = inspect.signature(hook_function.function) - parameters: tuple[Parameter, ...] = tuple(signature.parameters.values()) - accepts_var_keyword: bool = any( - parameter.kind is Parameter.VAR_KEYWORD for parameter in parameters - ) - keyword_parameter_names: frozenset[str] = frozenset( - parameter.name - for parameter in parameters - if parameter.kind in (Parameter.POSITIONAL_OR_KEYWORD, Parameter.KEYWORD_ONLY) - and parameter.name not in _HOOK_CONTEXT_PARAMETER_NAMES - ) - - context_conflicts: tuple[str, ...] = tuple( - sorted( - kwarg_name - for kwarg_name in hook_entry.kwargs - if kwarg_name in _HOOK_CONTEXT_PARAMETER_NAMES - ) - ) - if context_conflicts: - conflict: str = context_conflicts[0] - raise CompileInputError( - f"model '{model_name}' {hook_key}[{hook_index}] python(\"{hook_entry.name}\") " - f"argument '{conflict}' conflicts with reserved context parameter '{conflict}'. " - "Rename the hook argument; context parameters are injected by SQLBuild." - ) - - provider_conflicts: tuple[str, ...] = tuple( - sorted(kwarg_name for kwarg_name in hook_entry.kwargs if kwarg_name in provider_names) - ) - if provider_conflicts: - conflict = provider_conflicts[0] - raise CompileInputError( - f"model '{model_name}' {hook_key}[{hook_index}] python(\"{hook_entry.name}\") " - f"argument '{conflict}' conflicts with provider injection for parameter " - f"'{conflict}'. Rename the hook argument or remove it to let SQLBuild inject " - "the provider." - ) - - unknown_kwargs: tuple[str, ...] = tuple( - sorted( - kwarg_name - for kwarg_name in hook_entry.kwargs - if kwarg_name not in keyword_parameter_names and not accepts_var_keyword - ) - ) - if unknown_kwargs: - accepted_kwargs: str = _format_hook_parameter_names(keyword_parameter_names) - raise CompileInputError( - f"model '{model_name}' {hook_key}[{hook_index}] python(\"{hook_entry.name}\") " - f"has unknown argument(s): {', '.join(unknown_kwargs)}. " - f"Accepted configured arguments: {accepted_kwargs}" - ) - - required_positional_only: tuple[str, ...] = tuple( - parameter.name - for parameter in parameters - if parameter.kind is Parameter.POSITIONAL_ONLY - and parameter.default is Parameter.empty - and parameter.name not in _HOOK_CONTEXT_PARAMETER_NAMES + return expand_effective_vars( + _native.effective_config_vars((project_config, local_config), target_config, cli_vars) ) - if required_positional_only: - raise CompileInputError( - f"model '{model_name}' {hook_key}[{hook_index}] python(\"{hook_entry.name}\") " - "cannot be configured because the hook function has required positional-only " - f"parameter(s): {', '.join(required_positional_only)}. " - "Use keyword-capable parameters for values supplied from MODEL hooks." - ) - - missing_kwargs: tuple[str, ...] = tuple( - parameter.name - for parameter in parameters - if parameter.kind in (Parameter.POSITIONAL_OR_KEYWORD, Parameter.KEYWORD_ONLY) - and parameter.default is Parameter.empty - and parameter.name not in _HOOK_CONTEXT_PARAMETER_NAMES - and parameter.name not in provider_names - and parameter.name not in hook_entry.kwargs - ) - if missing_kwargs: - raise CompileInputError( - f"model '{model_name}' {hook_key}[{hook_index}] python(\"{hook_entry.name}\") " - f"is missing required argument(s): {', '.join(missing_kwargs)}" - ) - - -def _format_hook_parameter_names(parameter_names: frozenset[str]) -> str: - if not parameter_names: - return "none" - return ", ".join(sorted(parameter_names)) - - -def expand_sql_macros_in_value( - *, - value: object, - context: _HookExpansionContext, - hook_key: str | None = None, - hook_index: int | None = None, -) -> object: - """Recursively expand SQL interpolation and macros in hook container shapes.""" - - if isinstance(value, str): - if _HOOK_TEMPLATE_PATTERN.search(value) is not None: - hook_label: str = _format_sql_hook_label(hook_key=hook_key, hook_index=hook_index) - raise CompileInputError( - f"{hook_label} in '{context.file_path}' uses unsupported ${{...}} template syntax. " - "Use @@CTX:..., @@ENV:..., or @@project_var inside SQL hooks." - ) - declaration_context: DeclarationResolutionContext = ( - context.declaration_expansion.declarations - ) - if context.declaration_expansion.resolver is not None: - declaration_context = resolve_declaration_context( - resolver=context.declaration_expansion.resolver, - file_path=context.file_path, - ) - expansion: AuthoredSqlExpansionResult = expand_authored_sql_result( - sql=value, - file_path=context.file_path, - effective_vars=context.effective_vars, - context_values=context.context_values, - loaded_macros=context.loaded_macros, - macro_context=context.macro_context, - declaration_resolver=context.declaration_expansion.resolver, - value_renderer=context.declaration_expansion.value_renderer, - collection_rendering=context.declaration_expansion.collection_rendering, - declarations=replace(declaration_context, consumer=context.consumer), - ) - facts: _HookExpansionFacts = context.facts - facts.add(expansion.usages) - facts.add_references( - merge_call_site_references( - references=extract_sql_references( - sql=expansion.sql, - syntax=context.sql_lexical_syntax, - origin=context.reference_origin, - ), - argument_references=expansion.argument_references, - ) - ) - return expansion.sql - if isinstance(value, SqlHookEntry): - inline_start: int = len(context.facts.references) - expanded_statement: object = expand_sql_macros_in_value( - value=value.statement, - context=context, - hook_key=hook_key, - hook_index=hook_index, - ) - expanded_statement_text: str = str(expanded_statement) - _require_nonempty_sql_hook_payload( - statement=expanded_statement_text, - hook_label=_format_sql_hook_label(hook_key=hook_key, hook_index=hook_index), - file_path=context.file_path, - ) - return SqlHookEntry( - statement=expanded_statement_text, - name=value.name, - relative_path=value.relative_path, - definition_sql=value.definition_sql, - kwargs=value.kwargs, - description=value.description, - reads=context.facts.reads_since(inline_start), - ) - if isinstance(value, NamedSqlHookEntry): - hook_definition: DiscoveredSqlHookFile | None = context.sql_hook_definitions.get(value.name) - if hook_definition is None: - known_hook_names: str = ( - ", ".join(sorted(context.sql_hook_definitions)) or "none discovered" - ) - hook_label = _format_named_sql_hook_label( - hook_name=value.name, - hook_key=hook_key, - hook_index=hook_index, - ) - raise CompileInputError( - f"{hook_label} in '{context.file_path}' references an unknown SQL hook. " - f"Discovered SQL hooks: {known_hook_names}. If this is an inline SQL " - 'statement, use inline_sql("...").' - ) - hook_facts: _HookExpansionFacts = context.facts - hook_facts.add( - named_declaration_usages( - resolver=context.declaration_expansion.resolver, - kind=DeclarationKind.SQL_HOOK, - name=value.name, - consumer=context.consumer, - consumer_path=context.file_path, - ) - ) - rendered_statement: str = render_parameterized_sql( - sql=hook_definition.sql_body, - arguments=value.kwargs, - owner_label=( - f"{context.file_path} " - + _format_named_sql_hook_label( - hook_name=value.name, - hook_key=hook_key, - hook_index=hook_index, - ) - ), - definition_label=(f"SQL hook '{value.name}' in {hook_definition.relative_path}"), - reject_unused=True, - ) - named_start: int = len(hook_facts.references) - expanded_statement = expand_sql_macros_in_value( - value=rendered_statement, - context=replace( - context, - file_path=hook_definition.file_path, - consumer=DeclarationIdentity(DeclarationKind.SQL_HOOK, value.name), - ), - hook_key=hook_key, - hook_index=hook_index, - ) - expanded_statement_text: str = str(expanded_statement) - _require_nonempty_sql_hook_payload( - statement=expanded_statement_text, - hook_label=_format_named_sql_hook_label( - hook_name=value.name, - hook_key=hook_key, - hook_index=hook_index, - ), - file_path=context.file_path, - ) - return SqlHookEntry( - statement=expanded_statement_text, - name=value.name, - relative_path=hook_definition.relative_path, - definition_sql=hook_definition.sql_body, - kwargs=dict(value.kwargs), - description=hook_definition.description, - reads=hook_facts.reads_since(named_start), - ) - if isinstance(value, PythonHookEntry): - return value - if isinstance(value, list): - return [ - expand_sql_macros_in_value( - value=item, - context=context, - hook_key=hook_key, - hook_index=index, - ) - for index, item in enumerate(value) - ] - if isinstance(value, tuple): - return tuple( - expand_sql_macros_in_value( - value=item, - context=context, - hook_key=hook_key, - hook_index=index, - ) - for index, item in enumerate(value) - ) - return value - - -def _format_sql_hook_label(*, hook_key: str | None, hook_index: int | None) -> str: - if hook_key is None: - return 'inline_sql("...") hook' - if hook_index is None: - return f'{hook_key} inline_sql("...")' - return f'{hook_key}[{hook_index}] inline_sql("...")' - - -def _format_named_sql_hook_label( - *, hook_name: str, hook_key: str | None, hook_index: int | None -) -> str: - if hook_key is None: - return f'sql("{hook_name}") hook' - if hook_index is None: - return f'{hook_key} sql("{hook_name}")' - return f'{hook_key}[{hook_index}] sql("{hook_name}")' - - -def _require_nonempty_sql_hook_payload(*, statement: str, hook_label: str, file_path: Path) -> None: - if not statement.strip(): - raise CompileInputError( - f"{hook_label} in '{file_path}' must render a non-empty SQL payload" - ) def _index_sql_hook_definitions( @@ -1619,35 +483,3 @@ def _index_sql_hook_definitions( ) definitions[hook_file.name] = hook_file return definitions - - -def _authored_model_location( - *, - model_file: DiscoveredSqlModelFile, - sql: str, - offset: int, - passes: tuple[tuple[ExpansionSpan, ...], ...], -) -> SourceLocation | None: - """Map an offset in expanded model SQL to its authored file location, if it is authored.""" - - contents: str = model_file.contents - body_start: int | None = get_model_query_start( - contents=contents, query_sql=model_file.query_sql - ) - if body_start is None and contents.lstrip().startswith(model_file.query_sql): - body_start = len(contents) - len(contents.lstrip()) - mapped: MappedOffset = map_through_passes(offset=offset, passes=passes) - authored_offset: int = (body_start or 0) + mapped.offset - if ( - body_start is None - or mapped.generated - or offset >= len(sql) - or authored_offset >= len(contents) - or contents[authored_offset] != sql[offset] - ): - return None - return SourceLocation( - path=model_file.relative_path, - line=contents.count("\n", 0, authored_offset) + 1, - column=authored_offset - (contents.rfind("\n", 0, authored_offset) + 1) + 1, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 6ae2824615..3fb7baa190 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -7,9 +7,6 @@ from pathlib import Path from types import CodeType -from sqlbuild.compiler.compile._helpers.attachment.scope_relationships import ( - extract_scope_relationship_facts, -) from sqlbuild.compiler.compile._helpers.render.declarations import ( build_declaration_scope_resolver, ) @@ -18,6 +15,7 @@ validate_authored_test_cte_names, ) from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache +from sqlbuild.compiler.compile.constants import SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( DeclarationScopeBuild, @@ -28,7 +26,7 @@ from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index -from sqlbuild.compiler.scopes.models import RelationshipFact, ScopeIndex, ScopeLookup +from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -76,16 +74,13 @@ def _build_native_declaration_scope( if native.has_scoped_relationship_declarations and ( discovered_inputs.test_files or discovered_inputs.scenario_files ): - facts: tuple[RelationshipFact, ...] - fault: str | None - facts, fault = extract_scope_relationship_facts( + faults: tuple[tuple[Path, str], ...] = native.grant_relationships( discovered_inputs=discovered_inputs, - sql_lexical_syntax=sql_lexical_syntax, - scan_cache=scan_cache, + syntax=sql_lexical_syntax, + scan_store=(scan_cache.opened(), SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM), ) - if fault is not None: - raise CompileInputError(fault) - native.grant(facts) + if faults: + raise CompileInputError(faults[0][1]) index: ScopeIndex = native.index_with_relationships() lookup: ScopeLookup = native.lookup(index=index) return DeclarationScopeBuild( diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py deleted file mode 100644 index 78ea48e8a6..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ /dev/null @@ -1,632 +0,0 @@ -"""Attachment helpers for building pre-semantic compile inputs.""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery -from sqlbuild.compiler.attachments.constants import ( - HEADER_ARGUMENTS_STAGE, - HEADER_METADATA_STAGE, - HEADER_PYTHON_VALUES_STAGE, - HEADER_RETURNS_STAGE, - HEADER_START_STAGE, -) -from sqlbuild.compiler.attachments.main._parse_native_function_header import ( - parse_native_function_header, -) -from sqlbuild.compiler.attachments.main._resolve_native_function_namespace import ( - resolve_native_function_namespace, -) -from sqlbuild.compiler.attachments.models import ( - NativeFunctionHeader, - NativeFunctionNamespace, - NativeFunctionNamespaceInputs, -) -from sqlbuild.compiler.compile._helpers.analysis.validation import ( - validate_function_sql_syntax, -) -from sqlbuild.compiler.compile._helpers.attachment.references import ( - build_known_function_names, - build_known_ref_names, - build_known_seed_names, - build_known_source_names, - build_known_table_function_names, - validate_function_references, -) -from sqlbuild.compiler.compile._helpers.config.namespace_validation import ( - validate_preserved_logical_namespace, -) -from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( - merge_call_site_references, -) -from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references -from sqlbuild.compiler.compile._helpers.render.context_templates import expand_config_templates -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics -from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_expansion -from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_result -from sqlbuild.compiler.compile.constants import ( - PRESERVE_TARGET_VALUE, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - AuthoredSqlExpansionResult, - CompileSqlFunctionInput, - CompileSqlReference, - DeclarationExpansionContext, - FunctionArgument, - FunctionReturnColumn, - InferredColumn, - LoadedMacro, - MacroContext, - SqlReferenceOrigin, -) -from sqlbuild.compiler.compile.types import ( - FunctionLanguage, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredPythonFunctionFile, - DiscoveredSqlFunctionFile, -) -from sqlbuild.compiler.scopes.models import ResourceIdentity -from sqlbuild.compiler.scopes.types import ResourceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import ( - DefaultsConfig, - SettingsConfig, - TargetConfig, -) - - -@dataclass(frozen=True) -class _PythonFunctionBuildContext: - """Run-constant inputs for building one Python function compile input.""" - - effective_vars: dict[str, object] - effective_settings: SettingsConfig - adapter_name: str - no_sql_validation: bool - logical_database: str | None - logical_schema: str | None - target_database: str | None - target_schema: str | None - target_config: TargetConfig | None - python_functions_inherit_default_namespace: bool - - -def build_sql_function_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - effective_vars: dict[str, object], - effective_settings: SettingsConfig, - target_config: TargetConfig | None, - macro_context: MacroContext, - loaded_macros: dict[str, LoadedMacro], - declaration_expansion: DeclarationExpansionContext, - sql_lexical_syntax: SqlLexicalSyntax, - no_sql_validation: bool = False, - python_functions_inherit_default_namespace: bool = True, -) -> tuple[CompileSqlFunctionInput, ...]: - """Attach and validate SQL function metadata.""" - - adapter_name: str = macro_context.adapter_name - known_model_names: set[str] = build_known_ref_names(discovered_inputs) - known_seed_names: set[str] = build_known_seed_names(discovered_inputs) - known_source_names: set[str] = build_known_source_names(discovered_inputs) - known_function_names: set[str] = build_known_function_names(discovered_inputs) - known_table_function_names: set[str] = build_known_table_function_names(discovered_inputs) - logical_database, logical_schema = _resolve_function_logical_namespace( - defaults=discovered_inputs.project_config.defaults, - effective_vars=effective_vars, - ) - target_database, target_schema = _resolve_function_target_overrides( - target_config=target_config, - effective_vars=effective_vars, - ) - known_names: set[str] = set() - function_inputs: list[CompileSqlFunctionInput] = [] - function_file: DiscoveredSqlFunctionFile - for function_file in discovered_inputs.sql_function_files: - function_name: str = function_file.file_path.stem - if function_name in known_names: - raise CompileInputError(f"Duplicate SQL function name '{function_name}'") - known_names.add(function_name) - header_values: dict[str, object] = function_file.header_values - native_header: NativeFunctionHeader - arguments: tuple[FunctionArgument, ...] - returns: str - return_columns: tuple[FunctionReturnColumn, ...] - native_header, arguments, returns, return_columns = _sql_function_header( - function_file=function_file, effective_vars=effective_vars - ) - raw_database: object | None = header_values.get("database") - raw_schema: object | None = header_values.get("schema") - function_logical_database: str | None = ( - _expand_function_header_value( - raw_value=raw_database, - effective_vars=effective_vars, - context_label=f"SQL function {function_file.relative_path} database", - ) - if isinstance(raw_database, str) - else logical_database - ) - function_logical_schema: str | None = ( - _expand_function_header_value( - raw_value=raw_schema, - effective_vars=effective_vars, - context_label=f"SQL function {function_file.relative_path} schema", - ) - if isinstance(raw_schema, str) - else logical_schema - ) - validate_preserved_logical_namespace( - resource_label=f"SQL function '{function_name}'", - logical_database=function_logical_database, - logical_schema=function_logical_schema, - target_config=target_config, - ) - function_database, function_schema = _sql_function_namespace( - header_database=function_logical_database if isinstance(raw_database, str) else None, - header_schema=function_logical_schema if isinstance(raw_schema, str) else None, - defaults=(logical_database, logical_schema), - targets=(target_database, target_schema), - ) - scoped_declarations: DeclarationExpansionContext = resolve_declaration_expansion( - context=declaration_expansion, - file_path=function_file.file_path, - resource=ResourceIdentity(ResourceKind.FUNCTION, function_name), - ) - expansion: AuthoredSqlExpansionResult = expand_authored_sql_result( - sql=function_file.body_sql, - file_path=function_file.file_path, - effective_vars=effective_vars, - loaded_macros=loaded_macros, - macro_context=macro_context, - declarations=scoped_declarations.declarations, - declaration_resolver=scoped_declarations.resolver, - value_renderer=scoped_declarations.value_renderer, - collection_rendering=scoped_declarations.collection_rendering, - ) - expanded_body_sql: str = expansion.sql - reject_cursor_intrinsics( - sql=expanded_body_sql, - context=f"SQL function '{function_name}'", - ) - if effective_settings.sql_analysis and not no_sql_validation: - argument: FunctionArgument - for argument in arguments: - validate_native_type( - type_sql=argument.type, - adapter_name=adapter_name, - context=( - f"SQL function {function_file.relative_path} argument '{argument.name}'" - ), - ) - if return_columns: - return_column: FunctionReturnColumn - for return_column in return_columns: - validate_native_type( - type_sql=return_column.type, - adapter_name=adapter_name, - context=( - f"SQL function {function_file.relative_path} return column " - f"'{return_column.name}'" - ), - ) - else: - validate_native_type( - type_sql=returns, - adapter_name=adapter_name, - context=f"SQL function {function_file.relative_path} return type", - ) - validate_function_sql_syntax( - body_sql=expanded_body_sql, - function_name=function_name, - file_path=function_file.file_path, - ) - if return_columns: - _validate_table_function_output_contract( - body_sql=expanded_body_sql, - return_columns=return_columns, - function_file=function_file, - ) - references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=extract_sql_references( - sql=expanded_body_sql, - syntax=sql_lexical_syntax, - origin=SqlReferenceOrigin( - file_path=function_file.file_path, - relative_path=function_file.relative_path, - contents=function_file.contents, - ), - ), - argument_references=expansion.argument_references, - ) - validate_function_references( - references=references, - function_file=function_file, - known_model_names=known_model_names, - known_seed_names=known_seed_names, - known_source_names=known_source_names, - known_function_names=known_function_names, - known_table_function_names=known_table_function_names, - ) - _raise_header_failure(header=native_header, stage=HEADER_METADATA_STAGE) - function_inputs.append( - CompileSqlFunctionInput( - function_file=function_file, - name=function_name, - arguments=arguments, - returns=returns, - body_sql=expanded_body_sql, - return_columns=return_columns, - references=references, - database=function_database, - schema=function_schema, - logical_database=function_logical_database, - logical_schema=function_logical_schema, - fingerprint_database=function_database, - fingerprint_schema=function_schema, - fingerprint_logical_database=function_logical_database, - fingerprint_logical_schema=function_logical_schema, - tags=native_header.tags, - description=native_header.description, - declaration_usages=expansion.usages, - ) - ) - python_context: _PythonFunctionBuildContext = _PythonFunctionBuildContext( - effective_vars=effective_vars, - effective_settings=effective_settings, - adapter_name=adapter_name, - no_sql_validation=no_sql_validation, - logical_database=logical_database, - logical_schema=logical_schema, - target_database=target_database, - target_schema=target_schema, - target_config=target_config, - python_functions_inherit_default_namespace=(python_functions_inherit_default_namespace), - ) - python_function_file: DiscoveredPythonFunctionFile - for python_function_file in discovered_inputs.python_function_files: - function_name = python_function_file.file_path.stem - if function_name in known_names: - raise CompileInputError(f"Duplicate function name '{function_name}'") - known_names.add(function_name) - function_inputs.append( - _build_python_function_input( - python_function_file=python_function_file, - context=python_context, - ) - ) - return tuple(function_inputs) - - -def _sql_function_header( - *, function_file: DiscoveredSqlFunctionFile, effective_vars: dict[str, object] -) -> tuple[ - NativeFunctionHeader, tuple[FunctionArgument, ...], str, tuple[FunctionReturnColumn, ...] -]: - native_header: NativeFunctionHeader = parse_native_function_header( - header_values=function_file.header_values, - python=False, - relative_path=function_file.relative_path, - ) - _raise_header_failure(header=native_header, stage=HEADER_START_STAGE) - arguments: tuple[FunctionArgument, ...] = _expanded_native_arguments( - header=native_header, - label=f"SQL function {function_file.relative_path}", - effective_vars=effective_vars, - ) - _raise_header_failure(header=native_header, stage=HEADER_ARGUMENTS_STAGE) - native_returns: tuple[str, tuple[FunctionReturnColumn, ...]] = _expanded_native_sql_returns( - header=native_header, - relative_path=function_file.relative_path, - effective_vars=effective_vars, - ) - _raise_header_failure(header=native_header, stage=HEADER_RETURNS_STAGE) - return (native_header, arguments, *native_returns) - - -def _sql_function_namespace( - *, - header_database: str | None, - header_schema: str | None, - defaults: tuple[str | None, str | None], - targets: tuple[str | None, str | None], -) -> tuple[str | None, str | None]: - namespace: NativeFunctionNamespace = resolve_native_function_namespace( - NativeFunctionNamespaceInputs( - header_database=header_database, - header_schema=header_schema, - default_database=defaults[0], - default_schema=defaults[1], - target_database=targets[0], - target_schema=targets[1], - python=False, - inherit_default_namespace=True, - ) - ) - return namespace.database, namespace.schema - - -def _raise_header_failure(*, header: NativeFunctionHeader, stage: str) -> None: - if header.failure is not None and header.failure[0] == stage: - raise CompileInputError(header.failure[1]) - - -def _expanded_native_arguments( - *, header: NativeFunctionHeader, label: str, effective_vars: dict[str, object] -) -> tuple[FunctionArgument, ...]: - return tuple( - FunctionArgument( - name=argument.name, - type=_expand_function_header_value( - raw_value=argument.type_text, - effective_vars=effective_vars, - context_label=f"{label} argument '{argument.raw_name}' type", - ), - ) - for argument in header.arguments - ) - - -def _expanded_native_sql_returns( - *, header: NativeFunctionHeader, relative_path: Path, effective_vars: dict[str, object] -) -> tuple[str, tuple[FunctionReturnColumn, ...]]: - if header.return_columns is None: - return _expand_function_header_value( - raw_value=cast(str, header.returns), - effective_vars=effective_vars, - context_label=f"SQL function {relative_path} returns", - ), () - return "TABLE", tuple( - FunctionReturnColumn( - name=column.name, - type=_expand_function_header_value( - raw_value=column.type_text, - effective_vars=effective_vars, - context_label=( - f"SQL function {relative_path} return column '{column.raw_name}' type" - ), - ), - ) - for column in header.return_columns - ) - - -def _validate_table_function_output_contract( - *, - body_sql: str, - return_columns: tuple[FunctionReturnColumn, ...], - function_file: DiscoveredSqlFunctionFile, -) -> None: - inferred_columns: tuple[InferredColumn, ...] | None = infer_native_query_columns( - queries=(NativeColumnQuery(sql=body_sql, mode="parse"),), - profile=ExpressionInferenceProfile(), - )[0].columns - if not inferred_columns: - return - declared_count: int = len(return_columns) - inferred_count: int = len(inferred_columns) - if declared_count == inferred_count: - return - raise CompileInputError( - f"SQL table function {function_file.relative_path} declares {declared_count} return " - f"columns but its query produces {inferred_count}" - ) - - -def _build_python_function_input( - *, - python_function_file: DiscoveredPythonFunctionFile, - context: _PythonFunctionBuildContext, -) -> CompileSqlFunctionInput: - effective_vars: dict[str, object] = context.effective_vars - function_name: str = python_function_file.file_path.stem - header_values: dict[str, object] = python_function_file.header_values - native_header: NativeFunctionHeader = parse_native_function_header( - header_values=header_values, - python=True, - relative_path=python_function_file.relative_path, - ) - _raise_header_failure(header=native_header, stage=HEADER_START_STAGE) - arguments: tuple[FunctionArgument, ...] = _expanded_native_arguments( - header=native_header, - label=f"Python function {python_function_file.relative_path}", - effective_vars=effective_vars, - ) - _raise_header_failure(header=native_header, stage=HEADER_ARGUMENTS_STAGE) - returns: str = _expand_function_header_value( - raw_value=cast(str, native_header.returns), - effective_vars=effective_vars, - context_label=f"Python function {python_function_file.relative_path} returns", - ) - _raise_header_failure(header=native_header, stage=HEADER_PYTHON_VALUES_STAGE) - runtime_version: str = cast(str, native_header.runtime_version) - entry_point: str = cast(str, native_header.entry_point) - packages: tuple[str, ...] = native_header.packages - raw_database: object | None = header_values.get("database") - raw_schema: object | None = header_values.get("schema") - inherit: bool = context.python_functions_inherit_default_namespace - function_logical_database: str | None - if isinstance(raw_database, str): - function_logical_database = _expand_function_header_value( - raw_value=raw_database, - effective_vars=effective_vars, - context_label=f"Python function {python_function_file.relative_path} database", - ) - else: - function_logical_database = context.logical_database if inherit else None - function_logical_schema: str | None - if isinstance(raw_schema, str): - function_logical_schema = _expand_function_header_value( - raw_value=raw_schema, - effective_vars=effective_vars, - context_label=f"Python function {python_function_file.relative_path} schema", - ) - else: - function_logical_schema = context.logical_schema if inherit else None - namespace: NativeFunctionNamespace = resolve_native_function_namespace( - NativeFunctionNamespaceInputs( - header_database=function_logical_database if isinstance(raw_database, str) else None, - header_schema=function_logical_schema if isinstance(raw_schema, str) else None, - default_database=context.logical_database, - default_schema=context.logical_schema, - target_database=context.target_database, - target_schema=context.target_schema, - python=True, - inherit_default_namespace=inherit, - ) - ) - validate_preserved_logical_namespace( - resource_label=f"Python function '{function_name}'", - logical_database=namespace.fingerprint_logical_database, - logical_schema=namespace.fingerprint_logical_schema, - target_config=context.target_config, - ) - if context.effective_settings.sql_analysis and not context.no_sql_validation: - argument: FunctionArgument - for argument in arguments: - validate_native_type( - type_sql=argument.type, - adapter_name=context.adapter_name, - context=( - f"Python function {python_function_file.relative_path} " - f"argument '{argument.name}'" - ), - ) - validate_native_type( - type_sql=returns, - adapter_name=context.adapter_name, - context=f"Python function {python_function_file.relative_path} return type", - ) - _raise_header_failure(header=native_header, stage=HEADER_METADATA_STAGE) - return CompileSqlFunctionInput( - function_file=python_function_file, - name=function_name, - arguments=arguments, - returns=returns, - body_sql=python_function_file.body_python, - database=namespace.database, - schema=namespace.schema, - logical_database=function_logical_database, - logical_schema=function_logical_schema, - fingerprint_database=namespace.fingerprint_database, - fingerprint_schema=namespace.fingerprint_schema, - fingerprint_logical_database=namespace.fingerprint_logical_database, - fingerprint_logical_schema=namespace.fingerprint_logical_schema, - language=FunctionLanguage.PYTHON, - runtime_version=runtime_version, - entry_point=entry_point, - packages=packages, - tags=native_header.tags, - description=native_header.description, - ) - - -def _expand_function_header_value( - *, raw_value: str, effective_vars: dict[str, object], context_label: str -) -> str: - return str( - expand_config_templates( - value=raw_value, - variables=effective_vars, - context_values={}, - context_label=context_label, - allow_context=False, - preserve_context_tokens=True, - preserve_unknown_context=False, - ) - ) - - -def validate_native_type(*, type_sql: str, adapter_name: str, context: str) -> None: - """Validate an adapter-native type string with SQL analysis when a dialect is known.""" - - error: str | None = _native.function_type_error(type_sql, adapter_name, context) - if error is not None: - raise CompileInputError(error) - - -def _resolve_function_logical_namespace( - *, - defaults: DefaultsConfig, - effective_vars: dict[str, object], -) -> tuple[str | None, str | None]: - raw_database: str | None = ( - defaults.function_database if defaults.function_database is not None else defaults.database - ) - database: str | None = ( - _expand_function_header_value( - raw_value=raw_database, - effective_vars=effective_vars, - context_label="default function database", - ) - if raw_database is not None - else None - ) - raw_schema: str | None = ( - defaults.function_schema if defaults.function_schema is not None else defaults.schema - ) - schema: str | None = ( - _expand_function_header_value( - raw_value=raw_schema, - effective_vars=effective_vars, - context_label="default function schema", - ) - if raw_schema is not None - else None - ) - return database, schema - - -def _resolve_function_target_overrides( - *, target_config: TargetConfig | None, effective_vars: dict[str, object] -) -> tuple[str | None, str | None]: - if target_config is None: - return None, None - database: str | None = ( - _expand_function_environment_value( - raw_value=target_config.database, - effective_vars=effective_vars, - context_label="target database", - ) - if target_config.database is not None - else None - ) - schema: str | None = ( - _expand_function_environment_value( - raw_value=target_config.schema, - effective_vars=effective_vars, - context_label="target schema", - ) - if target_config.schema is not None - else None - ) - return database, schema - - -def _expand_function_environment_value( - *, raw_value: str, effective_vars: dict[str, object], context_label: str -) -> str | None: - if raw_value == PRESERVE_TARGET_VALUE: - return None - return str( - expand_config_templates( - value=raw_value, - variables=effective_vars, - context_values={}, - context_label=context_label, - allow_context=False, - preserve_context_tokens=True, - preserve_unknown_context=False, - ) - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index be34fdd96c..dc4dff9a3c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -4,23 +4,9 @@ import sys import unicodedata -from dataclasses import replace -from pathlib import Path from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.authored_values.main._optional_named_string import optional_named_string -from sqlbuild.compiler.compile._helpers.attachment.references import validate_external_reference -from sqlbuild.compiler.compile._helpers.audit_factories.core import ( - merge_validated_model_audits, - parse_model_header_audit_factories, -) -from sqlbuild.compiler.compile._helpers.config.dynamic_columns import ( - parse_dynamic_column_families, -) -from sqlbuild.compiler.compile._helpers.config.model_validation import ( - validate_column_migration_config, -) from sqlbuild.compiler.compile._helpers.config.retention import resolve_time_travel_retention from sqlbuild.compiler.compile._helpers.config.table_type import resolve_table_type from sqlbuild.compiler.compile._helpers.render.templating import ( @@ -29,21 +15,18 @@ from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment from sqlbuild.compiler.compile.constants import ( MODEL_AUDIT_OVERRIDE_KEYS, - MODEL_HEADER_METADATA_KEYS, ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileModelConfig, - CompileModelInput, - CompileSqlReference, ModelResourceNames, ModelValidationRequest, - ModelValidatorContext, NativeModelConfigInputs, NativeModelConfigSession, ) +from sqlbuild.compiler.compile.types import ModelInput +from sqlbuild.compiler.compiled_project.main._native_config_classes import native_config_classes from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditFactory, DiscoveredSchemaFile, DiscoveredSqlModelFile, ModelSchemaDeclaration, @@ -52,28 +35,15 @@ SqlHookEntry, ) from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.model_config.main._parse_native_header_metadata import ( - parse_native_header_metadata, -) -from sqlbuild.compiler.model_config.models import NativeHeaderMetadata from sqlbuild.compiler.path_defaults.main._select import select_path_default from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, ProjectConfig, - ResolvedTableType, - ResolvedTimeTravelRetention, - SchemaAuditInstance, - SchemaColumn, - SchemaDynamicColumnFamily, SchemaModelEntry, - SettingsConfig, - SourceLocation, TargetConfig, ) -from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TimeTravelRetentionSource _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) _TABLE_BACKED_MATERIALIZATIONS: tuple[str, ...] = ( @@ -82,7 +52,6 @@ MaterializationType.SNAPSHOT, ) _PYTHON_VERSION: tuple[int, int] = (sys.version_info[0], sys.version_info[1]) -_MATERIALIZED_CONFIG_KEY: str = "materialized" type _BuiltNamespace = tuple[str | None, bool, str | None] type _StorageOverrides = tuple[tuple[int | None, bool] | None, str | None] type _BuiltConfig = tuple[ @@ -194,219 +163,6 @@ def _as_string_list(value: object) -> list[str]: return [] -def build_model_header_schema_entry( - *, - model_name: str, - model_header_values: dict[str, object], - file_path: Path, - column_locations: dict[str, SourceLocation] | None = None, - model_schema_columns: tuple[SchemaColumn, ...] | None = None, - model_schema_name: str | None = None, - model_schema_description: str | None = None, - audit_factories: tuple[DiscoveredAuditFactory, ...] = (), -) -> SchemaModelEntry | None: - """Normalize model-owned MODEL(...) metadata into the existing schema entry shape.""" - - raw_description: object | None = model_header_values.get("description") - raw_columns: object | None = model_header_values.get("columns") - raw_dynamic_columns: object | None = model_header_values.get("dynamic_columns") - raw_audits: object | None = model_header_values.get("audits") - raw_audit_factories: object | None = model_header_values.get("audit_factories") - if ( - raw_description is None - and raw_columns is None - and raw_dynamic_columns is None - and raw_audits is None - and raw_audit_factories is None - and model_schema_columns is None - ): - return None - - model_description: str | None = optional_named_string( - raw_value=raw_description, - file_path=file_path, - label="model", - key="description", - error_class=CompileInputError, - ) - description: str | None = model_description or model_schema_description - native: NativeHeaderMetadata = parse_native_header_metadata( - raw_columns=raw_columns, - raw_audits=raw_audits, - column_locations=column_locations or {}, - file_path=file_path, - ) - if native.columns_error is not None: - raise native_config_error(error=native.columns_error) - local_columns: tuple[SchemaColumn, ...] = native.columns - columns: tuple[SchemaColumn, ...] = _merge_model_schema_columns( - model_name=model_name, - file_path=file_path, - model_schema_columns=model_schema_columns, - local_columns=local_columns, - ) - dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = parse_dynamic_column_families( - raw_value=raw_dynamic_columns, - model_name=model_name, - file_path=file_path, - ) - if native.audits_error is not None: - raise native_config_error(error=native.audits_error) - audits: tuple[SchemaAuditInstance, ...] = native.audits - generated_audits: tuple[SchemaAuditInstance, ...] = parse_model_header_audit_factories( - raw_audit_factories=raw_audit_factories, - file_path=file_path, - model_name=model_name, - audit_factories=audit_factories, - ) - audits = merge_validated_model_audits( - direct_audits=audits, - generated_audits=generated_audits, - model_name=model_name, - file_path=file_path, - ) - validate_column_migration_config( - config_values=model_header_values, model_name=model_name, columns=columns - ) - type_enforcement: bool | None = ( - True if any(column.type is not None for column in columns) or dynamic_columns else None - ) - return SchemaModelEntry( - name=model_name, - model_schema=model_schema_name, - description=description, - type_enforcement=type_enforcement, - columns=columns, - dynamic_columns=dynamic_columns, - audits=audits, - ) - - -def strip_model_header_metadata_from_config(config: CompileModelConfig) -> CompileModelConfig: - """Remove model metadata keys after they have been attached as schema metadata.""" - - filtered_values: dict[str, object] = { - key: value for key, value in config.values.items() if key not in MODEL_HEADER_METADATA_KEYS - } - if len(filtered_values) == len(config.values): - return config - return CompileModelConfig( - values=filtered_values, - model_header_keys=config.model_header_keys, - matched_path_default=config.matched_path_default, - logical_schema=config.logical_schema, - layer_schema=config.layer_schema, - logical_database=config.logical_database, - time_travel_retention=config.time_travel_retention, - table_type=config.table_type, - ) - - -def _merge_model_schema_columns( - *, - model_name: str, - file_path: Path, - model_schema_columns: tuple[SchemaColumn, ...] | None, - local_columns: tuple[SchemaColumn, ...], -) -> tuple[SchemaColumn, ...]: - if model_schema_columns is None: - return local_columns - merged_named_columns: list[SchemaColumn] = list(model_schema_columns) - named_index_by_name: dict[str, int] = { - column.name.lower(): index for index, column in enumerate(model_schema_columns) - } - additional_columns: list[SchemaColumn] = [] - local_column: SchemaColumn - for local_column in local_columns: - named_index: int | None = named_index_by_name.get(local_column.name.lower()) - if named_index is None: - additional_columns.append(local_column) - continue - named_column: SchemaColumn = merged_named_columns[named_index] - _validate_model_schema_audit_augmentation( - model_name=model_name, - file_path=file_path, - local_column=local_column, - named_column=named_column, - ) - merged_named_columns[named_index] = replace( - named_column, - audits=_merge_schema_audits( - inherited_audits=named_column.audits, - local_audits=local_column.audits, - ), - migrate_from=local_column.migrate_from, - ) - return (*merged_named_columns, *additional_columns) - - -def _merge_schema_audits( - *, - inherited_audits: tuple[SchemaAuditInstance, ...], - local_audits: tuple[SchemaAuditInstance, ...], -) -> tuple[SchemaAuditInstance, ...]: - merged_audits: list[SchemaAuditInstance] = list(inherited_audits) - audit: SchemaAuditInstance - for audit in local_audits: - if audit not in merged_audits: - merged_audits.append(audit) - return tuple(merged_audits) - - -def _validate_model_schema_audit_augmentation( - *, - model_name: str, - file_path: Path, - local_column: SchemaColumn, - named_column: SchemaColumn, -) -> None: - named_origin: str = ( - f"{named_column.location.path}:{named_column.location.line}" - if named_column.location is not None - else "the named schema" - ) - overridden_fields: list[str] = [] - if local_column.type is not None: - overridden_fields.append("type") - if local_column.nullable is not None: - overridden_fields.append("nullable") - if local_column.description is not None: - overridden_fields.append("description") - if overridden_fields: - raise CompileInputError( - f"model '{model_name}' in {file_path} cannot override " - f"{', '.join(overridden_fields)} for named-schema column '{local_column.name}' from " - f"{named_origin}; only audit augmentation is supported" - ) - if not local_column.audits and local_column.migrate_from is None: - raise CompileInputError( - f"model '{model_name}' in {file_path} redeclares named-schema column " - f"'{local_column.name}' from {named_origin} without audits or migrate_from; only " - "audit augmentation and column migrations are supported" - ) - - -def _resolve_model_schema( - *, - values: dict[str, object], - model_name: str, - public_model_schemas: dict[str, ModelSchemaDeclaration], -) -> ModelSchemaDeclaration | None: - raw_name: object | None = values.get("model_schema") - if raw_name is None: - return None - if not isinstance(raw_name, str) or not raw_name.strip(): - raise CompileInputError(f"model '{model_name}': model_schema must be a non-empty string") - declaration: ModelSchemaDeclaration | None = public_model_schemas.get(raw_name) - if declaration is None: - available: str = ", ".join(sorted(public_model_schemas)) or "none" - raise CompileInputError( - f"model '{model_name}' references unknown model_schema '{raw_name}'; " - f"available schemas: {available}" - ) - return declaration - - def has_resolved_model_description( *, model_file: DiscoveredSqlModelFile, @@ -517,52 +273,28 @@ def project_defaults_to_mapping(defaults: DefaultsConfig) -> dict[str, object]: return values -def find_schema_model_match( - *, - model_file: DiscoveredSqlModelFile, +def legacy_schema_rows( schema_files: tuple[DiscoveredSchemaFile, ...], -) -> tuple[SchemaModelEntry, DiscoveredSchemaFile] | None: - """Find the schema.yml model entry that applies to a discovered model file.""" +) -> list[tuple[str, list[str]]]: + """Each legacy schema file's relative path and the model names it declares.""" - model_name: str = model_file.file_path.stem - matching_entries: list[tuple[SchemaModelEntry, DiscoveredSchemaFile]] = [] - schema_file: DiscoveredSchemaFile + rows: list[tuple[str, list[str]]] = [] for schema_file in schema_files: - schema_directory: Path = schema_file.relative_path.parent - try: - model_file.relative_path.relative_to(schema_directory) - except ValueError: - continue - - schema_entry: SchemaModelEntry - for schema_entry in schema_file.model_entries: - if schema_entry.name == model_name: - matching_entries.append((schema_entry, schema_file)) - - if not matching_entries: - return None - if len(matching_entries) > 1: - matching_paths: str = ", ".join( - str(schema_file.relative_path) for _, schema_file in matching_entries + rows.append( + (str(schema_file.relative_path), [entry.name for entry in schema_file.model_entries]) ) - raise CompileInputError( - f"Model file {model_file.relative_path} matched multiple schema.yml declarations: " - f"{matching_paths}" - ) - return matching_entries[0] + return rows def validate_declared_schema_models_are_attached( *, - model_inputs: tuple[CompileModelInput, ...], + model_inputs: tuple[ModelInput, ...], schema_files: tuple[DiscoveredSchemaFile, ...], ) -> None: """Ensure every declared schema.yml model entry attaches within its directory scope.""" - attached_model_names: set[str] = { - model_input.schema_entry.name - for model_input in model_inputs - if model_input.schema_entry is not None + attached_model_names: set[str | None] = { + model_input.attached_schema_name for model_input in model_inputs } schema_file: DiscoveredSchemaFile for schema_file in schema_files: @@ -592,40 +324,6 @@ def _bool_from_dict(*, values: dict[str, object], key: str) -> bool: return raw -def _is_sql_validation_enabled(*, project_setting: bool, model_config: CompileModelConfig) -> bool: - """Apply the per-model unified SQL analysis override to the project setting.""" - - raw_analysis: object | None = model_config.values.get("sql_analysis") - raw_legacy: object | None = model_config.values.get("sql_validation") - if raw_analysis is not None and raw_legacy is not None and raw_analysis != raw_legacy: - raise CompileInputError( - "MODEL sql_analysis conflicts with legacy sql_validation", - code="P003", - ) - raw: object | None = raw_analysis if raw_analysis is not None else raw_legacy - if isinstance(raw, bool): - return raw - return project_setting - - -def _model_sql_validation_gate( - *, - effective_settings: SettingsConfig, - no_sql_validation: bool, - model_config: CompileModelConfig, -) -> bool: - """Apply the unified project/model/CLI SQL analysis gate.""" - - return ( - effective_settings.sql_analysis - and not no_sql_validation - and _is_sql_validation_enabled( - project_setting=effective_settings.sql_analysis, - model_config=model_config, - ) - ) - - def native_model_config_session( *, inputs: NativeModelConfigInputs, names: ModelResourceNames ) -> NativeModelConfigSession: @@ -679,19 +377,6 @@ def native_model_config_session( ) -def native_path_default( - *, session: NativeModelConfigSession, model_file: DiscoveredSqlModelFile -) -> str | None: - """Select a model's path default natively, raising Python's error for equal matches.""" - - selected: str | _native.NativeConfigError | None = session.builder.path_default( - str(model_file.relative_path) - ) - if isinstance(selected, _native.NativeConfigError): - raise native_config_error(error=selected) - return selected - - def build_native_model_config( *, session: NativeModelConfigSession, @@ -700,47 +385,14 @@ def build_native_model_config( ) -> CompileModelConfig: """Build a model's effective config natively, raising its first config error.""" - built: _BuiltConfig | _native.NativeConfigError = session.builder.build( - model_file.header_values, matched_path_default, model_file.file_path.stem + built: object = session.builder.model_config( + model_file.header_values, + (matched_path_default, model_file.file_path.stem), + native_config_classes(session=session, record_template_reads=record_template_reads), ) if isinstance(built, _native.NativeConfigError): raise native_config_error(error=built) - values, header_keys, namespace, overrides, reads = built - logical_schema, layer_schema_configured, logical_database = namespace - retention_override, table_type_override = overrides - record_template_reads(tuple(reads)) - materialized: object | None = values.get(_MATERIALIZED_CONFIG_KEY) - inherited_retention, inherited_table_type = session.inherited_storage[ - materialized - if isinstance(materialized, str) and materialized in _TABLE_BACKED_MATERIALIZATIONS - else None - ] - return CompileModelConfig( - values=values, - model_header_keys=header_keys, - matched_path_default=matched_path_default, - logical_schema=logical_schema, - layer_schema=logical_schema if layer_schema_configured else None, - logical_database=logical_database, - time_travel_retention=( - inherited_retention - if retention_override is None - else ResolvedTimeTravelRetention( - desired_days=retention_override[0], - unmanaged=retention_override[1], - source=TimeTravelRetentionSource.MODEL, - ) - ), - table_type=( - inherited_table_type - if table_type_override is None - else ResolvedTableType( - value=TableType(table_type_override), - source=TableTypeSource.MODEL, - declared=True, - ) - ), - ) + return cast(CompileModelConfig, built) def native_model_validation( @@ -778,79 +430,3 @@ def native_validation_error( """Return a native validator error, unmarked because validators read macro-expanded SQL.""" return native_config_error(error=error, values=values) - - -def validate_model_config( - *, context: ModelValidatorContext, request: ModelValidationRequest -) -> None: - """Validate one model natively, raising the first validator error.""" - - external_errors: list[Exception | None] = [ - _external_reference_error( - context=context, model_file=request.model_file, reference=reference - ) - for reference in request.references - ] - outcome: int | _native.NativeConfigError | None = native_model_validation( - session=context.native_config, - request=request, - rejected=[error is not None for error in external_errors], - ) - _raise_validation_outcome( - outcome=outcome, external_errors=external_errors, values=request.config.values - ) - - -def validate_model_references( - *, - context: ModelValidatorContext, - model_file: DiscoveredSqlModelFile, - references: tuple[CompileSqlReference, ...], -) -> None: - """Check references alone, as hook references are, raising the first problem.""" - - external_errors: list[Exception | None] = [ - _external_reference_error(context=context, model_file=model_file, reference=reference) - for reference in references - ] - outcome: int | _native.NativeConfigError | None = context.native_config.validator.references( - (model_file.file_path.stem, str(model_file.relative_path)), - [ - (str(reference.ref_kind), reference.ref_name, error is not None) - for reference, error in zip(references, external_errors, strict=True) - ], - ) - _raise_validation_outcome(outcome=outcome, external_errors=external_errors, values={}) - - -def _raise_validation_outcome( - *, - outcome: int | _native.NativeConfigError | None, - external_errors: list[Exception | None], - values: dict[str, object], -) -> None: - if isinstance(outcome, _native.NativeConfigError): - raise native_validation_error(error=outcome, values=values) - if outcome is not None: - raise cast(Exception, external_errors[outcome]) - - -def _external_reference_error( - *, - context: ModelValidatorContext, - model_file: DiscoveredSqlModelFile, - reference: CompileSqlReference, -) -> Exception | None: - """The error the dbt manifest gives a `__dbt_ref`, or None when it accepts the reference.""" - - if reference.ref_kind != SqlReferenceKind.DBT_REF: - return None - try: - validate_external_reference( - reference=reference, - model_file=model_file, - external_sql_reference_resolver=context.external_sql_reference_resolver, - ) - except CompileInputError as error: - return error - return None diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py b/src/sqlbuild/compiler/compile/_helpers/attachment/references.py index e527371794..4238a34d8c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/references.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/references.py @@ -3,226 +3,9 @@ from __future__ import annotations from sqlbuild.compiler.compile.constants import TABLE_FUNCTION_RETURN_KEYS -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileModelInput, - CompileSqlFunctionInput, - CompileSqlReference, -) from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditFile, DiscoveredProjectInputs, - DiscoveredSqlFunctionFile, - DiscoveredSqlModelFile, ) -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind - - -def validate_external_reference( - *, - reference: CompileSqlReference, - model_file: DiscoveredSqlModelFile, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> None: - """Validate one `__dbt_ref` against the dbt manifest, which the external resolver owns.""" - - if external_sql_reference_resolver is None: - raise CompileInputError( - f"Model file {model_file.relative_path} uses " - f"{SqlReferenceKind.DBT_REF.example_call(reference.ref_name, quote='"')} " - "but no dbt manifest was found", - code="C214", - help=( - "Run dbt compile or configure dbt target_path so SQLBuild can read manifest.json." - ), - ) - external_sql_reference_resolver.validate_reference( - ref_kind=reference.ref_kind, - ref_name=reference.ref_name, - ref_package=reference.ref_package, - owner_relative_sql_path=model_file.relative_path, - ) - - -def validate_table_function_call_arities( - *, - model_inputs: tuple[CompileModelInput, ...], - sql_function_inputs: tuple[CompileSqlFunctionInput, ...], -) -> None: - """Validate managed table-function calls against declared argument counts.""" - - argument_counts: dict[str, int] = { - function_input.name: len(function_input.arguments) - for function_input in sql_function_inputs - if function_input.return_columns - } - model_input: CompileModelInput - for model_input in model_inputs: - validate_table_function_reference_arities( - references=model_input.references, - argument_counts=argument_counts, - owner=f"Model file {model_input.model_file.relative_path}", - ) - function_input: CompileSqlFunctionInput - for function_input in sql_function_inputs: - validate_table_function_reference_arities( - references=function_input.references, - argument_counts=argument_counts, - owner=f"SQL function file {function_input.function_file.relative_path}", - ) - - -def validate_table_function_reference_arities( - *, - references: tuple[CompileSqlReference, ...], - argument_counts: dict[str, int], - owner: str, -) -> None: - reference: CompileSqlReference - for reference in references: - if reference.ref_kind != SqlReferenceKind.TABLE_FUNCTION: - continue - expected_count: int | None = argument_counts.get(reference.ref_name) - if expected_count is None or reference.call_argument_count == expected_count: - continue - expected_label: str = "argument" if expected_count == 1 else "arguments" - raise CompileInputError( - f"{owner} table function '{reference.ref_name}' expects {expected_count} " - f"{expected_label} but received {reference.call_argument_count}" - ) - - -def validate_function_references( - *, - references: tuple[CompileSqlReference, ...], - function_file: DiscoveredSqlFunctionFile, - known_model_names: set[str], - known_seed_names: set[str], - known_source_names: set[str], - known_function_names: set[str], - known_table_function_names: set[str], -) -> None: - """Validate extracted SQL function refs against discovered project inputs.""" - - reference: CompileSqlReference - for reference in references: - if ( - reference.ref_kind == SqlReferenceKind.REF - and reference.ref_name not in known_model_names - ): - if reference.ref_name in known_seed_names: - raise CompileInputError( - f"SQL function file {function_file.relative_path} references seed " - f"'{reference.ref_name}' with {SqlReferenceKind.REF.placeholder_call('...')}. " - f"Use {SqlReferenceKind.SEED.example_call(reference.ref_name, quote='"')} " - f"for seed references; {SqlReferenceKind.REF.function_name} only " - "resolves models." - ) - raise CompileInputError( - f"SQL function file {function_file.relative_path} references unknown model " - f"'{reference.ref_name}'" - ) - if ( - reference.ref_kind == SqlReferenceKind.SEED - and reference.ref_name not in known_seed_names - ): - raise CompileInputError( - f"SQL function file {function_file.relative_path} references unknown seed " - f"'{reference.ref_name}'" - ) - if ( - reference.ref_kind == SqlReferenceKind.SOURCE - and reference.ref_name not in known_source_names - ): - raise CompileInputError( - f"SQL function file {function_file.relative_path} references unknown source " - f"'{reference.ref_name}'" - ) - if reference.ref_kind == SqlReferenceKind.DBT_REF: - raise CompileInputError( - f"SQL function file {function_file.relative_path} uses " - f"{SqlReferenceKind.DBT_REF.example_call(reference.ref_name, quote='"')} " - "but dbt refs are not supported yet; " - "support may be added in a future release" - ) - if ( - reference.ref_kind in {SqlReferenceKind.UDF, SqlReferenceKind.TABLE_FUNCTION} - and reference.ref_name not in known_function_names - ): - raise CompileInputError( - f"SQL function file {function_file.relative_path} references unknown SQL function " - f"'{reference.ref_name}'" - ) - if ( - reference.ref_kind == SqlReferenceKind.UDF - and reference.ref_name in known_table_function_names - ): - raise CompileInputError( - f"SQL function file {function_file.relative_path} references table function " - f"'{reference.ref_name}' with {SqlReferenceKind.UDF.placeholder_call()}; " - f"use {SqlReferenceKind.TABLE_FUNCTION.placeholder_call()}" - ) - if ( - reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION - and reference.ref_name not in known_table_function_names - ): - raise CompileInputError( - f"SQL function file {function_file.relative_path} references scalar function " - f"'{reference.ref_name}' with " - f"{SqlReferenceKind.TABLE_FUNCTION.placeholder_call()}; " - f"use {SqlReferenceKind.UDF.placeholder_call()} for scalar UDFs" - ) - - -def validate_audit_references( - *, - references: tuple[CompileSqlReference, ...], - audit_file: DiscoveredAuditFile, - known_model_names: set[str], - known_seed_names: set[str], - known_source_names: set[str], -) -> None: - """Validate extracted audit refs against discovered project inputs.""" - - reference: CompileSqlReference - for reference in references: - if reference.ref_kind == SqlReferenceKind.DBT_REF: - raise CompileInputError( - f"Audit file {audit_file.relative_path} may not use " - f"{SqlReferenceKind.DBT_REF.example_call(reference.ref_name, quote='"')}; " - "audit dbt model checks belong in dbt" - ) - if ( - reference.ref_kind == SqlReferenceKind.REF - and reference.ref_name not in known_model_names - ): - if reference.ref_name in known_seed_names: - raise CompileInputError( - f"Audit file {audit_file.relative_path} references seed '{reference.ref_name}' " - f"with {SqlReferenceKind.REF.placeholder_call('...')}. Use " - f"{SqlReferenceKind.SEED.example_call(reference.ref_name, quote='"')} for seed " - f"references; {SqlReferenceKind.REF.function_name} only resolves models." - ) - raise CompileInputError( - f"Audit file {audit_file.relative_path} references unknown model " - f"'{reference.ref_name}'" - ) - if ( - reference.ref_kind == SqlReferenceKind.SEED - and reference.ref_name not in known_seed_names - ): - raise CompileInputError( - f"Audit file {audit_file.relative_path} references unknown seed " - f"'{reference.ref_name}'" - ) - if ( - reference.ref_kind == SqlReferenceKind.SOURCE - and reference.ref_name not in known_source_names - ): - raise CompileInputError( - f"Audit file {audit_file.relative_path} references unknown source " - f"'{reference.ref_name}'" - ) def build_known_ref_names(discovered_inputs: DiscoveredProjectInputs) -> set[str]: diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py deleted file mode 100644 index b248915873..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ /dev/null @@ -1,504 +0,0 @@ -"""Expected-model declaration grants with tolerant relationship faults.""" - -from __future__ import annotations - -from collections.abc import Callable, Iterator, Sequence -from pathlib import Path -from typing import cast - -import orjson - -from sqlbuild.compiler.compile._helpers.render.macros import ( - find_macro_call_names, - find_nested_macro_call_names, -) -from sqlbuild.compiler.compile._helpers.scenarios.core import ( - extract_sql_scenario_expected_model_names, -) -from sqlbuild.compiler.compile._helpers.sql_tests.core import ( - extract_sql_test_expected_model_names, - extract_unclassified_sql_test_ctes, -) -from sqlbuild.compiler.compile._helpers.sql_tests.native import native_sql_test_ctes -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.constants import ( - MACRO_ACTUAL_TEST_CTE_NAME, - SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileSqlTestCte, - ScopeRelationshipBuild, - ScopeRelationshipFault, -) -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestFile -from sqlbuild.compiler.scopes.main._native_expected_model_names import ( - native_expected_model_names, -) -from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( - resolve_scope_declaration_visibility, -) -from sqlbuild.compiler.scopes.main._resolve_scope_path_visibility import ( - resolve_scope_path_visibility, -) -from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - DeclarationVisibility, - GrantRecord, - RelationshipFact, - ResourceIdentity, - ResourceRecord, - ScopeIndex, - ScopeLookup, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, GrantKind, ResourceKind, ScopeKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -type _SharedDeclarations = dict[tuple[str, str], tuple[DeclarationRecord, ...]] -type _BlockNames = tuple[str, ...] | Exception - - -def build_scope_relationship_grants( - *, - discovered_inputs: DiscoveredProjectInputs, - index: ScopeIndex, - sql_lexical_syntax: SqlLexicalSyntax, - scan_cache: SqlTestScanCache | None = None, -) -> ScopeRelationshipBuild: - """Return expected-model grants while retaining independent extraction faults.""" - - lookup: ScopeLookup = build_scope_lookup(index=index) - shared_declarations: _SharedDeclarations = _shared_declarations_by_directory(lookup=lookup) - test_grants, test_faults = _test_relationship_grants( - discovered_inputs=discovered_inputs, - lookup=lookup, - shared_declarations=shared_declarations, - scan_cache=scan_cache or SqlTestScanCache(cache_dir=None), - syntax=sql_lexical_syntax, - ) - scenario_grants, scenario_faults = _scenario_relationship_grants( - discovered_inputs=discovered_inputs, - lookup=lookup, - shared_declarations=shared_declarations, - syntax=sql_lexical_syntax, - ) - return ScopeRelationshipBuild( - grants=tuple(dict.fromkeys((*test_grants, *scenario_grants))), - faults=(*test_faults, *scenario_faults), - ) - - -def extract_scope_relationship_facts( - *, - discovered_inputs: DiscoveredProjectInputs, - sql_lexical_syntax: SqlLexicalSyntax, - scan_cache: SqlTestScanCache | None = None, -) -> tuple[tuple[RelationshipFact, ...], str | None]: - """Return the relationship names grants are resolved from, and the first extraction fault.""" - - facts: list[RelationshipFact] = [] - faults: list[ScopeRelationshipFault] = [] - names_by_file: list[list[_BlockNames]] = _expected_names_by_file( - test_files=discovered_inputs.test_files, - scan_cache=scan_cache or SqlTestScanCache(cache_dir=None), - syntax=sql_lexical_syntax, - scan_natively=True, - ) - macro_blocks: list[tuple[str, str]] = [] - for test_file in discovered_inputs.test_files: - macro_blocks.extend( - (block.sql_body, str(test_file.relative_path)) - for block in test_file.blocks - if block.mode is SqlTestMode.MACRO - ) - macro_test_ctes: Iterator[tuple[tuple[str, str], ...] | str] = iter( - native_sql_test_ctes(texts=macro_blocks, syntax=sql_lexical_syntax) - ) - for test_file, file_names in zip(discovered_inputs.test_files, names_by_file, strict=True): - for block, block_names in zip(test_file.blocks, file_names, strict=True): - native_ctes: tuple[tuple[str, str], ...] | str = ( - next(macro_test_ctes) if block.mode is SqlTestMode.MACRO else () - ) - try: - expected_names: tuple[str, ...] = _names_or_raise(block_names) - called_macros: tuple[str, ...] = find_nested_macro_call_names(block.sql_body) - facts.append( - RelationshipFact( - resource=ResourceIdentity( - ResourceKind.TEST, block.name or test_file.relative_path.stem - ), - expected_models=expected_names, - called_macros=called_macros, - tested_macros=( - _natively_tested_macro_names(native_ctes=native_ctes) - if block.mode is SqlTestMode.MACRO - else () - ), - ) - ) - except Exception as error: - faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) - scenario_names: list[tuple[str, ...] | str] = native_expected_model_names( - texts=[ - (scenario.sql_body, str(scenario.relative_path)) - for scenario in discovered_inputs.scenario_files - ], - scenario=True, - syntax=sql_lexical_syntax, - ) - for scenario, scanned in zip(discovered_inputs.scenario_files, scenario_names, strict=True): - if isinstance(scanned, str): - faults.append(ScopeRelationshipFault(scenario.relative_path, scanned)) - continue - facts.append( - RelationshipFact( - resource=ResourceIdentity(ResourceKind.SCENARIO, scenario.name), - expected_models=scanned, - ) - ) - return tuple(facts), faults[0].message if faults else None - - -def _expected_names_by_file( - *, - test_files: Sequence[DiscoveredSqlTestFile], - scan_cache: SqlTestScanCache, - syntax: SqlLexicalSyntax, - scan_natively: bool, -) -> list[list[_BlockNames]]: - """Return each block's expected-model names or extraction error, reusing whole stored files.""" - - stored: list[tuple[tuple[str, ...], ...] | None] = [ - scan_cache.read( - algorithm=SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, - syntax=syntax, - parts=_expected_names_parts(test_file), - decode=_stored_expected_names(block_count=len(test_file.blocks)), - ) - for test_file in test_files - ] - scanned: dict[tuple[int, int], _BlockNames] = ( - _scanned_expected_names(test_files=test_files, stored=stored, syntax=syntax) - if scan_natively - else {} - ) - names_by_file: list[list[_BlockNames]] = [] - for file_index, test_file in enumerate(test_files): - stored_names: tuple[tuple[str, ...], ...] | None = stored[file_index] - if stored_names is not None: - names_by_file.append(list(stored_names)) - continue - file_names: list[_BlockNames] = [] - for block_index, block in enumerate(test_file.blocks): - if scan_natively: - file_names.append(scanned.get((file_index, block_index), ())) - continue - try: - file_names.append( - extract_sql_test_expected_model_names( - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=syntax, - mode=block.mode, - ) - ) - except Exception as error: - file_names.append(error) - if not any(isinstance(names, Exception) for names in file_names): - scan_cache.write( - algorithm=SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, - syntax=syntax, - parts=_expected_names_parts(test_file), - value=orjson.dumps(file_names), - ) - names_by_file.append(file_names) - return names_by_file - - -def _expected_names_parts(test_file: DiscoveredSqlTestFile) -> list[str]: - parts: list[str] = [str(test_file.relative_path)] - for block in test_file.blocks: - parts.extend((block.sql_body, block.mode.value)) - return parts - - -def _stored_expected_names( - *, block_count: int -) -> Callable[[bytes], tuple[tuple[str, ...], ...] | None]: - """Decode one file's stored names, rejecting anything that is not their exact shape.""" - - def decode(value: bytes) -> tuple[tuple[str, ...], ...] | None: - try: - decoded: object = orjson.loads(value) - except orjson.JSONDecodeError: - return None - if not isinstance(decoded, list) or len(cast(list[object], decoded)) != block_count: - return None - blocks: list[tuple[str, ...]] = [] - for names in cast(list[object], decoded): - if not isinstance(names, list) or not all( - isinstance(name, str) for name in cast(list[object], names) - ): - return None - blocks.append(tuple(cast(list[str], names))) - return tuple(blocks) - - return decode - - -def _names_or_raise(names: _BlockNames) -> tuple[str, ...]: - if isinstance(names, Exception): - raise names - return names - - -def _scanned_expected_names( - *, - test_files: Sequence[DiscoveredSqlTestFile], - stored: Sequence[tuple[tuple[str, ...], ...] | None], - syntax: SqlLexicalSyntax, -) -> dict[tuple[int, int], _BlockNames]: - """Scan model-mode blocks of unstored files natively: their names or their error.""" - - pending: list[tuple[int, int]] = [] - for file_index, test_file in enumerate(test_files): - if stored[file_index] is not None: - continue - pending.extend( - (file_index, block_index) - for block_index, block in enumerate(test_file.blocks) - if block.mode is SqlTestMode.MODEL - ) - scanned: list[tuple[str, ...] | str] = native_expected_model_names( - texts=[ - ( - test_files[file_index].blocks[block_index].sql_body, - str(test_files[file_index].relative_path), - ) - for file_index, block_index in pending - ], - scenario=False, - syntax=syntax, - ) - return { - position: CompileInputError(names) if isinstance(names, str) else names - for position, names in zip(pending, scanned, strict=True) - } - - -def _natively_tested_macro_names( - *, - native_ctes: tuple[tuple[str, str], ...] | str, -) -> tuple[str, ...]: - """Macros a macro test's actual CTE calls, scanned natively.""" - - if isinstance(native_ctes, str): - raise CompileInputError(native_ctes) - actual_sql: str | None = next( - (body for name, body in native_ctes if name == MACRO_ACTUAL_TEST_CTE_NAME), None - ) - return () if actual_sql is None else find_macro_call_names(actual_sql) - - -def _test_relationship_grants( - *, - discovered_inputs: DiscoveredProjectInputs, - lookup: ScopeLookup, - shared_declarations: _SharedDeclarations, - scan_cache: SqlTestScanCache, - syntax: SqlLexicalSyntax, -) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: - grants: list[GrantRecord] = [] - faults: list[ScopeRelationshipFault] = [] - names_by_file: list[list[_BlockNames]] = _expected_names_by_file( - test_files=discovered_inputs.test_files, - scan_cache=scan_cache, - syntax=syntax, - scan_natively=False, - ) - for test_file, file_names in zip(discovered_inputs.test_files, names_by_file, strict=True): - for block, block_names in zip(test_file.blocks, file_names, strict=True): - try: - expected_names: tuple[str, ...] = _names_or_raise(block_names) - grants.extend( - _expected_model_grants( - lookup=lookup, - resource=ResourceIdentity( - ResourceKind.TEST, block.name or test_file.relative_path.stem - ), - expected_model_names=expected_names, - shared_declarations=shared_declarations, - called_macros=frozenset(find_nested_macro_call_names(block.sql_body)), - ) - ) - if block.mode is SqlTestMode.MACRO: - grants.extend( - _tested_macro_grants( - lookup=lookup, - resource=ResourceIdentity( - ResourceKind.TEST, block.name or test_file.relative_path.stem - ), - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=syntax, - ) - ) - except Exception as error: - faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) - return tuple(grants), tuple(faults) - - -def _scenario_relationship_grants( - *, - discovered_inputs: DiscoveredProjectInputs, - lookup: ScopeLookup, - shared_declarations: _SharedDeclarations, - syntax: SqlLexicalSyntax, -) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: - grants: list[GrantRecord] = [] - faults: list[ScopeRelationshipFault] = [] - for scenario in discovered_inputs.scenario_files: - try: - expected_names: tuple[str, ...] = extract_sql_scenario_expected_model_names( - sql=scenario.sql_body, file_label=str(scenario.relative_path), syntax=syntax - ) - grants.extend( - _expected_model_grants( - lookup=lookup, - resource=ResourceIdentity(ResourceKind.SCENARIO, scenario.name), - expected_model_names=expected_names, - shared_declarations=shared_declarations, - ) - ) - except Exception as error: - faults.append(ScopeRelationshipFault(scenario.relative_path, str(error))) - return tuple(grants), tuple(faults) - - -def _expected_model_grants( - *, - lookup: ScopeLookup, - resource: ResourceIdentity, - expected_model_names: tuple[str, ...], - shared_declarations: _SharedDeclarations, - called_macros: frozenset[str] = frozenset(), -) -> list[GrantRecord]: - grants: list[GrantRecord] = [] - for model_name in expected_model_names: - through: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, model_name) - for declaration in _shared_visible_declarations( - lookup=lookup, resource=through, shared=shared_declarations - ): - if ( - declaration.identity.kind is DeclarationKind.MACRO - and declaration.identity.name not in called_macros - ): - continue - grants.append( - GrantRecord( - resource=resource, - declaration=declaration.identity, - through=through, - ) - ) - return grants - - -def _shared_declarations_by_directory(*, lookup: ScopeLookup) -> _SharedDeclarations: - """Resolve shared non-private visibility once per model directory and ownership root.""" - - representatives: dict[tuple[str, str], ResourceIdentity] = {} - for identity, records in lookup.resources.items(): - if ( - identity.kind is ResourceKind.MODEL - and len(records) == 1 - and identity not in lookup.grants_by_resource - ): - representatives.setdefault(_directory_key(record=records[0]), identity) - return { - key: _resolve_shared_declarations(lookup=lookup, resource=identity) - for key, identity in representatives.items() - } - - -def _directory_key(*, record: ResourceRecord) -> tuple[str, str]: - return (Path(record.path).parent.as_posix(), record.ownership_root.path) - - -def _shared_visible_declarations( - *, lookup: ScopeLookup, resource: ResourceIdentity, shared: _SharedDeclarations -) -> tuple[DeclarationRecord, ...]: - """Return non-private declarations a resource sees, reusing its directory's resolution.""" - - records: tuple[ResourceRecord, ...] = lookup.resources.get(resource, ()) - if len(records) == 1 and resource not in lookup.grants_by_resource: - directory_declarations: tuple[DeclarationRecord, ...] | None = shared.get( - _directory_key(record=records[0]) - ) - if directory_declarations is not None: - return directory_declarations - return _resolve_shared_declarations(lookup=lookup, resource=resource) - - -def _resolve_shared_declarations( - *, lookup: ScopeLookup, resource: ResourceIdentity -) -> tuple[DeclarationRecord, ...]: - resolution: DeclarationVisibility = resolve_scope_declaration_visibility( - lookup=lookup, target=resource - ) - declarations: list[DeclarationRecord] = [] - for visible in resolution.visible: - records: tuple[DeclarationRecord, ...] = lookup.declarations.get(visible.declaration, ()) - if records and records[0].scope is not ScopeKind.PRIVATE: - declarations.append(records[0]) - return tuple(declarations) - - -def _tested_macro_grants( - *, - lookup: ScopeLookup, - resource: ResourceIdentity, - sql: str, - file_label: str, - syntax: SqlLexicalSyntax, -) -> list[GrantRecord]: - test_ctes: tuple[CompileSqlTestCte, ...] = extract_unclassified_sql_test_ctes( - sql=sql, file_label=file_label, syntax=syntax - ) - actual_cte: CompileSqlTestCte | None = next( - (cte for cte in test_ctes if cte.name == MACRO_ACTUAL_TEST_CTE_NAME), None - ) - if actual_cte is None: - return [] - tested_macro_names: tuple[str, ...] = find_macro_call_names(actual_cte.sql_body) - direct_resolution: DeclarationVisibility = resolve_scope_declaration_visibility( - lookup=lookup, target=resource - ) - directly_visible: frozenset[DeclarationIdentity] = frozenset( - visible.declaration for visible in direct_resolution.visible - ) - grants: list[GrantRecord] = [] - for macro_name in tested_macro_names: - records: tuple[DeclarationRecord, ...] = lookup.declarations.get( - DeclarationIdentity(DeclarationKind.MACRO, macro_name), () - ) - if not records: - continue - tested_macro: DeclarationRecord = records[0] - lexical_path: Path = Path(tested_macro.owning_path or ".") / "__macro_test__.sql" - visible, _inaccessible = resolve_scope_path_visibility(lookup=lookup, path=lexical_path) - for declaration in visible: - if declaration.scope is ScopeKind.PRIVATE or declaration.identity in directly_visible: - continue - grants.append( - GrantRecord( - resource=resource, - declaration=declaration.identity, - through=tested_macro.identity, - kind=GrantKind.TESTED_MACRO, - ) - ) - return grants diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py deleted file mode 100644 index fc28ee11ec..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sources.py +++ /dev/null @@ -1,329 +0,0 @@ -"""Attachment helpers for building pre-semantic compile inputs.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.compile._helpers.analysis.validation import ( - validate_source_expression_syntax, -) -from sqlbuild.compiler.compile._helpers.render.context_templates import expand_config_templates -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics -from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_expansion -from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_result -from sqlbuild.compiler.compile.models import ( - AuthoredSqlExpansionResult, - CompileSourceInput, - DeclarationExpansionContext, - DeclarationResolutionContext, - DeclarationScopeResolver, - LoadedMacro, - MacroContext, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSourceFile, -) -from sqlbuild.compiler.scopes.models import ResourceIdentity, UsageRecord, VisibilityRecord -from sqlbuild.compiler.scopes.types import ResourceKind, ScopeKind -from sqlbuild.spec.contracts.models import ( - SchemaAuditInstance, - SettingsConfig, - SourceColumnEntry, - SourceEntry, -) - - -def build_source_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - effective_vars: dict[str, object], - effective_settings: SettingsConfig, - macro_context: MacroContext, - loaded_macros: dict[str, LoadedMacro], - declaration_expansion: DeclarationExpansionContext, - no_sql_validation: bool = False, -) -> tuple[CompileSourceInput, ...]: - """Normalize discovered source declarations into one collection.""" - - source_inputs: list[CompileSourceInput] = [] - sql_validation_enabled: bool = effective_settings.sql_analysis and not no_sql_validation - resolver: DeclarationScopeResolver | None = declaration_expansion.resolver - default_contract: str | None = discovered_inputs.project_config.defaults.contract - reuse_file_scope: bool = ( - resolver is not None - and not any( - declaration.scope is ScopeKind.PRIVATE - for declaration in resolver.lookup.index.declarations - ) - and not any( - resource.kind is ResourceKind.SOURCE for resource in resolver.lookup.grants_by_resource - ) - ) - declarations_by_file: dict[Path, DeclarationExpansionContext] = {} - source_file: DiscoveredSourceFile - for source_file in discovered_inputs.source_files: - source_entry: SourceEntry - for raw_source_entry in source_file.source_entries: - effective_source_entry: SourceEntry = raw_source_entry - if raw_source_entry.contract is None and default_contract is not None: - effective_source_entry = replace( - raw_source_entry, - contract=default_contract, - ) - source_resource: ResourceIdentity = ResourceIdentity( - ResourceKind.SOURCE, raw_source_entry.name - ) - scoped_declarations: DeclarationExpansionContext | None = declarations_by_file.get( - source_file.file_path - ) - if scoped_declarations is not None: - scoped_declarations = _rebind_source_declarations( - context=scoped_declarations, consumer=source_resource - ) - else: - scoped_declarations = resolve_declaration_expansion( - context=declaration_expansion, - file_path=source_file.file_path, - resource=source_resource, - ) - if reuse_file_scope: - declarations_by_file[source_file.file_path] = scoped_declarations - source_entry, usages = expand_source_entry_templates( - source_entry=effective_source_entry, - file_path=source_file.file_path, - effective_vars=effective_vars, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_expansion=declaration_expansion, - scoped_declarations=scoped_declarations, - ) - source_expression: str | None = source_entry.expression - if source_expression is not None: - reject_cursor_intrinsics( - sql=source_expression, - context=f"Source expression '{source_entry.name}'", - ) - should_validate_expression: bool = ( - source_expression is not None and sql_validation_enabled - ) - if should_validate_expression and source_expression is not None: - validate_source_expression_syntax( - expression=source_expression, - source_name=source_entry.name, - file_path=source_file.file_path, - ) - source_inputs.append( - CompileSourceInput( - source_entry=source_entry, - source_file=source_file, - declaration_usages=usages, - ) - ) - return tuple(source_inputs) - - -def expand_source_entry_templates( - *, - source_entry: SourceEntry, - file_path: Path, - effective_vars: dict[str, object], - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - declaration_expansion: DeclarationExpansionContext, - scoped_declarations: DeclarationExpansionContext | None = None, -) -> tuple[SourceEntry, tuple[UsageRecord, ...]]: - """Apply config templating and SQL interpolation to source metadata.""" - - expression: str | None = None - usages: tuple[UsageRecord, ...] = () - if scoped_declarations is None: - scoped_declarations = resolve_declaration_expansion( - context=declaration_expansion, - file_path=file_path, - resource=ResourceIdentity(ResourceKind.SOURCE, source_entry.name), - ) - if source_entry.expression is not None: - expansion: AuthoredSqlExpansionResult = expand_authored_sql_result( - sql=source_entry.expression, - file_path=file_path, - effective_vars=effective_vars, - loaded_macros=loaded_macros, - macro_context=macro_context, - declarations=scoped_declarations.declarations, - declaration_resolver=scoped_declarations.resolver, - value_renderer=scoped_declarations.value_renderer, - collection_rendering=scoped_declarations.collection_rendering, - ) - expression = expansion.sql - usages = expansion.usages - expanded_entry: SourceEntry = replace( - source_entry, - database=_expand_source_template_value( - raw_value=source_entry.database, - effective_vars=effective_vars, - context_label=f"source {source_entry.name} database", - ), - schema=_expand_source_template_value( - raw_value=source_entry.schema, - effective_vars=effective_vars, - context_label=f"source {source_entry.name} schema", - ), - table=_expand_source_template_value( - raw_value=source_entry.table, - effective_vars=effective_vars, - context_label=f"source {source_entry.name} table", - ), - expression=expression, - description=_expand_source_template_value( - raw_value=source_entry.description, - effective_vars=effective_vars, - context_label=f"source {source_entry.name} description", - ), - meta=cast( - dict[str, object], - _expand_source_template_object( - value=source_entry.meta, - effective_vars=effective_vars, - context_label=f"source {source_entry.name} meta", - ), - ), - columns=tuple( - expand_source_column_templates( - source_name=source_entry.name, - column=column, - effective_vars=effective_vars, - ) - for column in source_entry.columns - ), - audits=tuple( - expand_schema_audit_instance_templates( - audit_instance=audit_instance, - effective_vars=effective_vars, - context_label=f"source {source_entry.name} audit {audit_instance.definition_name}", - ) - for audit_instance in source_entry.audits - ), - ) - return expanded_entry, usages - - -def _rebind_source_declarations( - *, context: DeclarationExpansionContext, consumer: ResourceIdentity -) -> DeclarationExpansionContext: - declarations: DeclarationResolutionContext = context.declarations - return replace( - context, - declarations=replace( - declarations, - consumer=consumer, - enum_visibility=_rebind_visibility( - visibility=declarations.enum_visibility, consumer=consumer - ), - constant_visibility=_rebind_visibility( - visibility=declarations.constant_visibility, consumer=consumer - ), - ), - ) - - -def _rebind_visibility( - *, - visibility: dict[str, tuple[VisibilityRecord, ...]], - consumer: ResourceIdentity, -) -> dict[str, tuple[VisibilityRecord, ...]]: - rebound: dict[str, tuple[VisibilityRecord, ...]] = {} - for name, records in visibility.items(): - rebound[name] = tuple(replace(record, resource=consumer) for record in records) - return rebound - - -def expand_source_column_templates( - *, source_name: str, column: SourceColumnEntry, effective_vars: dict[str, object] -) -> SourceColumnEntry: - return replace( - column, - type=_expand_source_template_value( - raw_value=column.type, - effective_vars=effective_vars, - context_label=f"source {source_name} column {column.name} type", - ), - description=_expand_source_template_value( - raw_value=column.description, - effective_vars=effective_vars, - context_label=f"source {source_name} column {column.name} description", - ), - meta=cast( - dict[str, object], - _expand_source_template_object( - value=column.meta, - effective_vars=effective_vars, - context_label=f"source {source_name} column {column.name} meta", - ), - ), - audits=tuple( - expand_schema_audit_instance_templates( - audit_instance=audit_instance, - effective_vars=effective_vars, - context_label=( - f"source {source_name} column {column.name} audit " - f"{audit_instance.definition_name}" - ), - ) - for audit_instance in column.audits - ), - ) - - -def expand_schema_audit_instance_templates( - *, - audit_instance: SchemaAuditInstance, - effective_vars: dict[str, object], - context_label: str, -) -> SchemaAuditInstance: - return replace( - audit_instance, - arguments=cast( - dict[str, object], - _expand_source_template_object( - value=audit_instance.arguments, - effective_vars=effective_vars, - context_label=f"{context_label} arguments", - ), - ), - description=_expand_source_template_value( - raw_value=audit_instance.description, - effective_vars=effective_vars, - context_label=f"{context_label} description", - ), - ) - - -def _expand_source_template_value( - *, raw_value: str | None, effective_vars: dict[str, object], context_label: str -) -> str | None: - if raw_value is None: - return None - return str( - _expand_source_template_object( - value=raw_value, - effective_vars=effective_vars, - context_label=context_label, - ) - ) - - -def _expand_source_template_object( - *, value: object, effective_vars: dict[str, object], context_label: str -) -> object: - return expand_config_templates( - value=value, - variables=effective_vars, - context_values={}, - context_label=context_label, - allow_context=False, - preserve_context_tokens=False, - preserve_unknown_context=False, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py deleted file mode 100644 index 512861d990..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ /dev/null @@ -1,875 +0,0 @@ -"""Attachment helpers for building pre-semantic compile inputs.""" - -from __future__ import annotations - -from dataclasses import dataclass, replace -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.attachments.main._native_test_target_catalog import ( - native_test_target_catalog, -) -from sqlbuild.compiler.compile._helpers.attachment.references import ( - build_known_function_names, - build_known_ref_names, - build_known_seed_names, - build_known_source_names, - build_known_table_function_names, - validate_table_function_reference_arities, -) -from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics -from sqlbuild.compiler.compile._helpers.render.declarations import ( - declaration_usage_records, - resolve_declaration_expansion, - usage_visibility, -) -from sqlbuild.compiler.compile._helpers.render.macros import ( - find_macro_call_names, -) -from sqlbuild.compiler.compile._helpers.render.parameters import expand_test_parameters -from sqlbuild.compiler.compile._helpers.render.sql_vars import ( - expand_authored_sql_result, -) -from sqlbuild.compiler.compile._helpers.scenarios.core import extract_sql_scenario_ctes -from sqlbuild.compiler.compile._helpers.sql_tests.core import ( - complete_omitted_ceremonial_select, - extract_assertion_target_model_names, -) -from sqlbuild.compiler.compile._helpers.sql_tests.extraction_errors import ( - located_extraction_error, - report_authored_invalid_calls, -) -from sqlbuild.compiler.compile._helpers.sql_tests.helper_ctes import ( - report_test_without_target_model, - report_unresolvable_test_references, - sql_test_reads, -) -from sqlbuild.compiler.compile._helpers.sql_tests.native import ( - extract_expanded_sql_tests, - extract_unexpanded_sql_test, -) -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestExtractionError -from sqlbuild.compiler.compile.models import ( - AuthoredSqlExpansionResult, - CompileDirectLogicSqlTestCtes, - CompileDirectLogicSqlTestInputPayload, - CompileModelSqlTestCtes, - CompileModelSqlTestInputPayload, - CompileSqlFunctionInput, - CompileSqlReference, - CompileSqlScenarioCte, - CompileSqlScenarioCtes, - CompileSqlScenarioInput, - CompileSqlTestCtes, - CompileSqlTestInput, - DeclarationExpansionContext, - DeclarationResolutionContext, - DeclarationScopeResolver, - LoadedMacro, - MacroContext, - SqlReferenceOrigin, - SqlTestReads, -) -from sqlbuild.compiler.compile.types import ( - SqlTestMode, -) -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredProjectInputs, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestCase, - DiscoveredSqlTestFile, - EnumDeclaration, -) -from sqlbuild.compiler.profiling.main.record import record_compile_timing -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - ResourceIdentity, - UsageRecord, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ResourceKind, ScopeKind, UsageKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -@dataclass(frozen=True) -class _ExpandedSqlTest: - test_file: DiscoveredSqlTestFile - test_block: DiscoveredSqlTestBlock - sql_body: str - mode: SqlTestMode - tested_resource_names: tuple[str, ...] - declaration_usages: tuple[UsageRecord, ...] - parent_name: str - test_case: DiscoveredSqlTestCase | None - - -def build_test_inputs_with_cache( - *, - discovered_inputs: DiscoveredProjectInputs, - effective_vars: dict[str, object], - macro_context: MacroContext, - loaded_macros: dict[str, LoadedMacro], - declaration_expansion: DeclarationExpansionContext, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - sql_function_inputs: tuple[CompileSqlFunctionInput, ...], - sql_lexical_syntax: SqlLexicalSyntax, - scan_cache: SqlTestScanCache, -) -> tuple[CompileSqlTestInput, ...]: - """Build SQL test inputs inside the compile timing boundary, then save the scan store.""" - - with record_compile_timing("test_input_compile_ms"): - test_inputs: tuple[CompileSqlTestInput, ...] = build_test_inputs( - discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=loaded_macros, - declaration_expansion=declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_function_inputs=sql_function_inputs, - sql_lexical_syntax=sql_lexical_syntax, - scan_cache=scan_cache, - ) - scan_cache.save() - return test_inputs - - -def build_test_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - effective_vars: dict[str, object] | None = None, - macro_context: MacroContext, - loaded_macros: dict[str, LoadedMacro], - declaration_expansion: DeclarationExpansionContext, - sql_lexical_syntax: SqlLexicalSyntax, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - sql_function_inputs: tuple[CompileSqlFunctionInput, ...] = (), - scan_cache: SqlTestScanCache | None = None, -) -> tuple[CompileSqlTestInput, ...]: - """Build compile-time test inputs from discovered SQL-native test blocks.""" - - vars_for_substitution: dict[str, object] = effective_vars or {} - known_model_names: set[str] = build_known_ref_names(discovered_inputs) - if external_sql_reference_resolver is not None: - known_model_names.update( - external_sql_reference_resolver.extend_sql_test_model_names( - known_model_names=known_model_names - ) - ) - known_seed_names: set[str] = build_known_seed_names(discovered_inputs) - known_source_names: set[str] = build_known_source_names(discovered_inputs) - if external_sql_reference_resolver is not None: - known_seed_names.update( - external_sql_reference_resolver.extend_sql_test_seed_names( - known_seed_names=known_seed_names - ) - ) - known_source_names.update( - external_sql_reference_resolver.extend_sql_test_source_names( - known_source_names=known_source_names - ) - ) - known_function_names: set[str] = build_known_function_names(discovered_inputs) - known_table_function_names: set[str] = build_known_table_function_names(discovered_inputs) - table_function_argument_counts: dict[str, int] = { - function_input.name: len(function_input.arguments) - for function_input in sql_function_inputs - if function_input.return_columns - } - test_inputs: list[CompileSqlTestInput] = [] - expanded_tests: list[_ExpandedSqlTest] = [] - resolver: DeclarationScopeResolver | None = declaration_expansion.resolver - reuse_parent_scope: bool = ( - resolver is not None - and not any( - declaration.scope is ScopeKind.PRIVATE - for declaration in resolver.lookup.index.declarations - ) - and not any( - resource.kind is ResourceKind.TEST for resource in resolver.lookup.grants_by_resource - ) - ) - declarations_by_parent: dict[Path, DeclarationExpansionContext] = {} - test_file: DiscoveredSqlTestFile - for test_file in discovered_inputs.test_files: - test_block: DiscoveredSqlTestBlock - for test_block in test_file.blocks: - resource: ResourceIdentity = ResourceIdentity( - ResourceKind.TEST, test_block.name or test_file.relative_path.stem - ) - parent: Path = test_file.file_path.parent - scoped_declarations: DeclarationExpansionContext | None = declarations_by_parent.get( - parent - ) - if scoped_declarations is not None: - scoped_declarations = _project_test_declaration_consumer( - context=scoped_declarations, consumer=resource - ) - else: - scoped_declarations = resolve_declaration_expansion( - context=declaration_expansion, - file_path=test_file.file_path, - resource=resource, - ) - if reuse_parent_scope: - declarations_by_parent[parent] = scoped_declarations - case_variants: tuple[DiscoveredSqlTestCase | None, ...] = test_block.cases or (None,) - test_case: DiscoveredSqlTestCase | None - for test_case in case_variants: - parameter_sql: str = test_block.sql_body - if test_case is not None: - parameter_sql, used_parameters = expand_test_parameters( - sql=test_block.sql_body, - file_path=test_file.file_path, - values=test_case.values, - value_renderer=scoped_declarations.value_renderer, - test_name=test_block.name or test_file.file_path.stem, - case_name=test_case.name, - ) - unused_parameters: tuple[str, ...] = tuple( - parameter.name - for parameter in test_block.parameters - if parameter.name not in used_parameters - ) - if unused_parameters: - raise CompileInputError( - f"SQL test '{test_block.name or test_file.file_path.stem}' declares " - f"unused parameters: {', '.join(unused_parameters)} in case " - f"'{test_case.name}'", - ) - expanded_test_block: DiscoveredSqlTestBlock = ( - test_block - if parameter_sql is test_block.sql_body - else replace(test_block, sql_body=parameter_sql) - ) - test_mode: SqlTestMode = test_block.mode - tested_resource_names: tuple[str, ...] = () - if test_mode in {SqlTestMode.MACRO, SqlTestMode.UDF, SqlTestMode.TABLE_FN}: - raw_test_ctes: CompileSqlTestCtes = _validate_raw_direct_logic_test_ctes( - sql=expanded_test_block.sql_body, - test_block=test_block, - test_file=test_file, - test_mode=test_mode, - syntax=sql_lexical_syntax, - ) - tested_resource_names = _infer_tested_direct_logic_resource_names( - raw_test_ctes=raw_test_ctes, - test_file=test_file, - loaded_macros=loaded_macros, - known_function_names=known_function_names, - known_table_function_names=known_table_function_names, - table_function_argument_counts=table_function_argument_counts, - syntax=sql_lexical_syntax, - ) - expansion: AuthoredSqlExpansionResult = expand_authored_sql_result( - sql=parameter_sql, - file_path=test_file.file_path, - effective_vars=vars_for_substitution, - loaded_macros=loaded_macros, - macro_context=macro_context, - declarations=( - replace(scoped_declarations.declarations, consumer=None) - if test_mode is SqlTestMode.MACRO - else scoped_declarations.declarations - ), - declaration_resolver=scoped_declarations.resolver, - value_renderer=scoped_declarations.value_renderer, - collection_rendering=scoped_declarations.collection_rendering, - ) - expanded_sql_body: str = complete_omitted_ceremonial_select( - sql=expansion.sql, - syntax=sql_lexical_syntax, - ) - reject_cursor_intrinsics( - sql=expanded_sql_body, - context=f"SQL test '{test_block.name or test_file.file_path.stem}'", - ) - expanded_tests.append( - _ExpandedSqlTest( - test_file=test_file, - test_block=test_block, - sql_body=expanded_sql_body, - mode=test_mode, - tested_resource_names=tested_resource_names, - declaration_usages=( - _macro_test_declaration_usages( - sql=parameter_sql, - resource=resource, - declarations=scoped_declarations.declarations, - ) - if test_mode is SqlTestMode.MACRO - else expansion.usages - ), - parent_name=test_block.name or test_file.relative_path.stem, - test_case=test_case, - ) - ) - try: - test_ctes_batch: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( - tests=tuple( - (test.sql_body, str(test.test_file.relative_path), test.mode) - for test in expanded_tests - ), - scan_cache=scan_cache, - syntax=sql_lexical_syntax, - ) - except SqlTestExtractionError as error: - failed: _ExpandedSqlTest = expanded_tests[error.test_index] - raise located_extraction_error( - error=error, - test_file=failed.test_file, - test_block=failed.test_block, - sql=failed.sql_body, - ) from None - target_catalog: _native.SqlTestTargetCatalog = native_test_target_catalog( - models=known_model_names, - sources=known_source_names, - seeds=known_seed_names, - table_functions=known_table_function_names, - macros=loaded_macros, - ) - for test, test_ctes in zip(expanded_tests, test_ctes_batch, strict=True): - assertion_target_model_names: tuple[str, ...] = ( - _unmocked_assertion_targets( - payload=test_ctes.payload, test=test, syntax=sql_lexical_syntax - ) - if isinstance(test_ctes.payload, CompileModelSqlTestCtes) - else () - ) - validate_test_ctes( - test_ctes=test_ctes, - test_file=test.test_file, - known_model_names=known_model_names, - known_seed_names=known_seed_names, - known_source_names=known_source_names, - known_table_function_names=known_table_function_names, - loaded_macros=loaded_macros, - assertion_target_model_names=assertion_target_model_names, - target_catalog=target_catalog, - ) - reads: SqlTestReads = SqlTestReads(read_helper_names=(), reference_target_model_names=()) - if isinstance(test_ctes.payload, CompileModelSqlTestCtes): - if report_unresolvable_test_references( - payload=test_ctes.payload, - test_file=test.test_file, - test_block=test.test_block, - known_model_names=known_model_names, - syntax=sql_lexical_syntax, - ): - continue - reads = sql_test_reads(payload=test_ctes.payload, syntax=sql_lexical_syntax) - if ( - test.mode is SqlTestMode.MODEL - and not test_ctes.payload.expected_model_names - and not assertion_target_model_names - and not reads.reference_target_model_names - ): - report_test_without_target_model( - payload=test_ctes.payload, - test_file=test.test_file, - test_block=test.test_block, - syntax=sql_lexical_syntax, - ) - continue - test_inputs.append( - CompileSqlTestInput( - test_file=test.test_file, - test_block=test.test_block, - sql_body=test.sql_body, - mode=test.mode, - payload=_build_test_input_payload( - test_ctes=test_ctes, - tested_resource_names=test.tested_resource_names, - assertion_target_model_names=assertion_target_model_names, - reads=reads, - ), - declaration_usages=test.declaration_usages, - parent_name=test.parent_name, - case_name=test.test_case.name if test.test_case is not None else None, - case_index=test.test_case.case_index if test.test_case is not None else None, - parameter_schema=test.test_block.parameters, - parameter_values=test.test_case.values if test.test_case is not None else (), - ) - ) - return tuple(test_inputs) - - -def _unmocked_assertion_targets( - *, payload: CompileModelSqlTestCtes, test: _ExpandedSqlTest, syntax: SqlLexicalSyntax -) -> tuple[str, ...]: - """Return models the assertions call ``__ref()`` on that the test does not mock.""" - - targets: tuple[str, ...] = extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in payload.assertion_ctes), - syntax=syntax, - origin=_sql_file_reference_origin(test.test_file), - ) - return tuple(name for name in targets if name not in payload.mock_model_names) - - -def _project_test_declaration_consumer( - *, context: DeclarationExpansionContext, consumer: ResourceIdentity -) -> DeclarationExpansionContext: - return replace( - context, - declarations=replace(context.declarations, consumer=consumer), - ) - - -def _macro_test_declaration_usages( - *, - sql: str, - resource: ResourceIdentity, - declarations: DeclarationResolutionContext, -) -> tuple[UsageRecord, ...]: - usages: tuple[UsageRecord, ...] = declaration_usage_records( - sql=sql, - resource=resource, - declarations=declarations, - ) - macro_usages: list[UsageRecord] = [] - for name in find_macro_call_names(sql=sql): - if name not in declarations.macro_records: - continue - identity: DeclarationIdentity = declarations.macro_records[name].identity - visibility: tuple[VisibilityRecord, ...] = declarations.macro_visibility.get(name, ()) - visible_records: tuple[VisibilityRecord, ...] = usage_visibility( - visibility=visibility, consumer=resource - ) - if visible_records: - macro_usages.extend( - UsageRecord( - resource, - identity, - UsageKind.RUNTIME, - through=visible.through, - ) - for visible in visible_records - ) - else: - macro_usages.append(UsageRecord(resource, identity, UsageKind.RUNTIME)) - return tuple(dict.fromkeys((*usages, *macro_usages))) - - -def _build_test_input_payload( - *, - test_ctes: CompileSqlTestCtes, - tested_resource_names: tuple[str, ...], - assertion_target_model_names: tuple[str, ...], - reads: SqlTestReads, -) -> CompileModelSqlTestInputPayload | CompileDirectLogicSqlTestInputPayload: - match test_ctes.payload: - case CompileModelSqlTestCtes() as model_payload: - return CompileModelSqlTestInputPayload( - authored_ctes=model_payload.authored_ctes, - macro_mocks=model_payload.macro_mocks, - mock_model_names=model_payload.mock_model_names, - mock_source_names=model_payload.mock_source_names, - mock_seed_names=model_payload.mock_seed_names, - mock_dbt_ref_names=model_payload.mock_dbt_ref_names, - mock_table_function_names=model_payload.mock_table_function_names, - expected_ctes=model_payload.expected_ctes, - expected_model_names=model_payload.expected_model_names, - assertion_ctes=model_payload.assertion_ctes, - assertion_names=model_payload.assertion_names, - assertion_target_model_names=assertion_target_model_names, - read_helper_names=reads.read_helper_names, - reference_target_model_names=reads.reference_target_model_names, - ) - case CompileDirectLogicSqlTestCtes() as direct_logic_payload: - return CompileDirectLogicSqlTestInputPayload( - mode=direct_logic_payload.mode, - helper_ctes=direct_logic_payload.helper_ctes, - actual_cte=direct_logic_payload.actual_cte, - expected_cte=direct_logic_payload.expected_cte, - tested_resource_names=tested_resource_names, - ) - - -def _validate_raw_direct_logic_test_ctes( - *, - sql: str, - test_block: DiscoveredSqlTestBlock, - test_file: DiscoveredSqlTestFile, - test_mode: SqlTestMode, - syntax: SqlLexicalSyntax, -) -> CompileSqlTestCtes: - try: - test_ctes, invalid_calls = extract_unexpanded_sql_test( - sql=sql, file_label=str(test_file.relative_path), mode=test_mode, syntax=syntax - ) - except SqlTestExtractionError as error: - raise located_extraction_error( - error=error, test_file=test_file, test_block=test_block, sql=sql - ) from None - if not (invalid_calls and isinstance(test_ctes.payload, CompileDirectLogicSqlTestCtes)): - return test_ctes - reported: int = report_authored_invalid_calls( - test_file=test_file, - test_block=test_block, - actual_cte_name=test_ctes.payload.actual_cte.name, - syntax=syntax, - ) - if reported == 0: - for cte in (*test_ctes.payload.helper_ctes, test_ctes.payload.expected_cte): - _ = extract_sql_references( - sql=cte.sql_body, syntax=syntax, origin=_sql_file_reference_origin(test_file) - ) - return test_ctes - - -def _infer_tested_direct_logic_resource_names( - *, - raw_test_ctes: CompileSqlTestCtes, - test_file: DiscoveredSqlTestFile, - loaded_macros: dict[str, LoadedMacro], - public_enums: dict[str, EnumDeclaration] | None = None, - public_constants: dict[str, ConstantDeclaration] | None = None, - known_function_names: set[str], - known_table_function_names: set[str], - table_function_argument_counts: dict[str, int], - syntax: SqlLexicalSyntax, -) -> tuple[str, ...]: - if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): - raise CompileInputError( - f"SQL test file {test_file.relative_path} mode '{raw_test_ctes.mode.value}' must " - "define exactly one actual CTE and exactly one expected CTE", - ) - if raw_test_ctes.mode == SqlTestMode.UDF: - return _infer_tested_udf_names( - raw_test_ctes=raw_test_ctes, - test_file=test_file, - known_function_names=known_function_names, - known_table_function_names=known_table_function_names, - syntax=syntax, - ) - if raw_test_ctes.mode == SqlTestMode.TABLE_FN: - return _infer_tested_table_function_names( - raw_test_ctes=raw_test_ctes, - test_file=test_file, - known_function_names=known_function_names, - known_table_function_names=known_table_function_names, - table_function_argument_counts=table_function_argument_counts, - syntax=syntax, - ) - tested_macro_names: tuple[str, ...] = find_macro_call_names( - raw_test_ctes.payload.actual_cte.sql_body - ) - if not tested_macro_names: - raise CompileInputError( - f"SQL test file {test_file.relative_path} mode 'macro' must call at least one " - "macro in __macro_actual__", - ) - tested_macro_name: str - for tested_macro_name in tested_macro_names: - if tested_macro_name not in loaded_macros: - raise CompileInputError( - f"SQL test file {test_file.relative_path} references unknown macro " - f"'@{tested_macro_name}'", - ) - return tested_macro_names - - -def _infer_tested_udf_names( - *, - raw_test_ctes: CompileSqlTestCtes, - test_file: DiscoveredSqlTestFile, - known_function_names: set[str], - known_table_function_names: set[str], - syntax: SqlLexicalSyntax, -) -> tuple[str, ...]: - if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): - raise CompileInputError( - f"SQL test file {test_file.relative_path} mode 'udf' must define exactly one " - "__udf_actual__ CTE and exactly one __udf_expected__ CTE", - ) - references: tuple[CompileSqlReference, ...] = extract_sql_references( - sql=raw_test_ctes.payload.actual_cte.sql_body, - syntax=syntax, - origin=_sql_file_reference_origin(test_file), - ) - tested_udf_names: tuple[str, ...] = tuple( - dict.fromkeys( - reference.ref_name - for reference in references - if reference.ref_kind == SqlReferenceKind.UDF - ) - ) - if not tested_udf_names: - raise CompileInputError( - f"SQL test file {test_file.relative_path} mode 'udf' must call at least one " - "scalar UDF in __udf_actual__", - ) - tested_udf_name: str - for tested_udf_name in tested_udf_names: - if tested_udf_name not in known_function_names: - raise CompileInputError( - f"SQL test file {test_file.relative_path} references unknown SQL function " - f"'{tested_udf_name}'", - ) - if tested_udf_name in known_table_function_names: - raise CompileInputError( - f"SQL test file {test_file.relative_path} references table function " - f"'{tested_udf_name}' with {SqlReferenceKind.UDF.placeholder_call()}; use " - f"{SqlReferenceKind.TABLE_FUNCTION.placeholder_call()} for table functions", - ) - return tested_udf_names - - -def _infer_tested_table_function_names( - *, - raw_test_ctes: CompileSqlTestCtes, - test_file: DiscoveredSqlTestFile, - known_function_names: set[str], - known_table_function_names: set[str], - table_function_argument_counts: dict[str, int], - syntax: SqlLexicalSyntax, -) -> tuple[str, ...]: - if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): - raise CompileInputError( - f"SQL test file {test_file.relative_path} mode 'table_fn' must define exactly one " - "__table_fn_actual__ CTE and exactly one __table_fn_expected__ CTE", - ) - references: tuple[CompileSqlReference, ...] = extract_sql_references( - sql=raw_test_ctes.payload.actual_cte.sql_body, - syntax=syntax, - origin=_sql_file_reference_origin(test_file), - ) - validate_table_function_reference_arities( - references=references, - argument_counts=table_function_argument_counts, - owner=f"SQL test file {test_file.relative_path}", - ) - tested_table_function_names: tuple[str, ...] = tuple( - dict.fromkeys( - reference.ref_name - for reference in references - if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION - ) - ) - if not tested_table_function_names: - raise CompileInputError( - f"SQL test file {test_file.relative_path} mode 'table_fn' must call at least one " - "table function in __table_fn_actual__", - ) - tested_table_function_name: str - for tested_table_function_name in tested_table_function_names: - if tested_table_function_name not in known_function_names: - raise CompileInputError( - f"SQL test file {test_file.relative_path} references unknown SQL function " - f"'{tested_table_function_name}'", - ) - if tested_table_function_name not in known_table_function_names: - raise CompileInputError( - f"SQL test file {test_file.relative_path} references scalar SQL function " - f"'{tested_table_function_name}' with " - f"{SqlReferenceKind.TABLE_FUNCTION.placeholder_call()}; use " - f"{SqlReferenceKind.UDF.placeholder_call()} for scalar UDFs" - ) - return tested_table_function_names - - -def build_scenario_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - effective_vars: dict[str, object] | None = None, - macro_context: MacroContext, - loaded_macros: dict[str, LoadedMacro], - declaration_expansion: DeclarationExpansionContext, - sql_lexical_syntax: SqlLexicalSyntax, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> tuple[CompileSqlScenarioInput, ...]: - """Build compile-time scenario inputs from discovered SQL-native scenario files.""" - - vars_for_substitution: dict[str, object] = effective_vars or {} - known_source_names: set[str] = build_known_source_names(discovered_inputs) - if external_sql_reference_resolver is not None: - known_source_names.update( - external_sql_reference_resolver.extend_sql_test_source_names( - known_source_names=known_source_names - ) - ) - scenario_inputs: list[CompileSqlScenarioInput] = [] - source_catalog: _native.SqlTestTargetCatalog = native_test_target_catalog( - sources=known_source_names - ) - scenario_file: DiscoveredSqlScenarioFile - for scenario_file in discovered_inputs.scenario_files: - resource: ResourceIdentity = ResourceIdentity(ResourceKind.SCENARIO, scenario_file.name) - scoped_declarations: DeclarationExpansionContext = resolve_declaration_expansion( - context=declaration_expansion, - file_path=scenario_file.file_path, - resource=resource, - ) - expansion: AuthoredSqlExpansionResult = expand_authored_sql_result( - sql=scenario_file.sql_body, - file_path=scenario_file.file_path, - effective_vars=vars_for_substitution, - loaded_macros=loaded_macros, - macro_context=macro_context, - declarations=scoped_declarations.declarations, - declaration_resolver=scoped_declarations.resolver, - value_renderer=scoped_declarations.value_renderer, - collection_rendering=scoped_declarations.collection_rendering, - ) - expanded_sql_body: str = complete_omitted_ceremonial_select( - sql=expansion.sql, - syntax=sql_lexical_syntax, - ) - reject_cursor_intrinsics( - sql=expanded_sql_body, - context=f"SQL scenario '{scenario_file.file_path.stem}'", - ) - scenario_ctes: CompileSqlScenarioCtes = extract_sql_scenario_ctes( - sql=expanded_sql_body, - file_label=str(scenario_file.relative_path), - syntax=sql_lexical_syntax, - ) - _validate_scenario_source_references( - scenario_ctes=scenario_ctes, - scenario_file=scenario_file, - known_source_names=known_source_names, - syntax=sql_lexical_syntax, - target_catalog=source_catalog, - ) - assertion_target_model_names: tuple[str, ...] = extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in scenario_ctes.assertion_ctes), - syntax=sql_lexical_syntax, - origin=_sql_file_reference_origin(scenario_file), - ) - scenario_inputs.append( - CompileSqlScenarioInput( - scenario_file=scenario_file, - sql_body=expanded_sql_body, - authored_ctes=scenario_ctes.authored_ctes, - expected_ctes=scenario_ctes.expected_ctes, - assertion_ctes=scenario_ctes.assertion_ctes, - source_fixture_names=scenario_ctes.source_fixture_names, - ref_fixture_names=scenario_ctes.ref_fixture_names, - seed_fixture_names=scenario_ctes.seed_fixture_names, - dbt_ref_fixture_names=scenario_ctes.dbt_ref_fixture_names, - expected_model_names=scenario_ctes.expected_model_names, - assertion_names=scenario_ctes.assertion_names, - assertion_target_model_names=assertion_target_model_names, - target_model_names=tuple( - dict.fromkeys( - (*scenario_ctes.expected_model_names, *assertion_target_model_names) - ) - ), - declaration_usages=expansion.usages, - ) - ) - return tuple(scenario_inputs) - - -def _sql_file_reference_origin( - sql_file: DiscoveredSqlTestFile | DiscoveredSqlScenarioFile, -) -> SqlReferenceOrigin: - return SqlReferenceOrigin( - file_path=sql_file.file_path, - relative_path=sql_file.relative_path, - contents=sql_file.contents, - ) - - -def _validate_scenario_source_references( - *, - scenario_ctes: CompileSqlScenarioCtes, - scenario_file: DiscoveredSqlScenarioFile, - known_source_names: set[str], - syntax: SqlLexicalSyntax, - target_catalog: _native.SqlTestTargetCatalog | None = None, -) -> None: - cte_sources: list[tuple[str, bool, list[str]]] - extraction_error: CompileInputError | None - cte_sources, extraction_error = _scenario_cte_sources( - scenario_ctes=scenario_ctes, scenario_file=scenario_file, syntax=syntax - ) - error: str | None = ( - target_catalog or native_test_target_catalog(sources=known_source_names) - ).scenario_source_error(str(scenario_file.relative_path), cte_sources) - if error is not None: - raise CompileInputError(error) - if extraction_error is not None: - raise extraction_error - return - - -def _scenario_cte_sources( - *, - scenario_ctes: CompileSqlScenarioCtes, - scenario_file: DiscoveredSqlScenarioFile, - syntax: SqlLexicalSyntax, -) -> tuple[list[tuple[str, bool, list[str]]], CompileInputError | None]: - """Each check-then-fixture CTE's source references, up to the first CTE whose scan raises.""" - - checks: tuple[CompileSqlScenarioCte, ...] = ( - *scenario_ctes.expected_ctes, - *scenario_ctes.assertion_ctes, - ) - cte_sources: list[tuple[str, bool, list[str]]] = [] - for position, cte in enumerate((*checks, *scenario_ctes.authored_ctes)): - try: - references: tuple[CompileSqlReference, ...] = extract_sql_references( - sql=cte.sql_body, syntax=syntax, origin=_sql_file_reference_origin(scenario_file) - ) - except CompileInputError as error: - return cte_sources, error - cte_sources.append( - ( - cte.name, - position < len(checks), - [ - reference.ref_name - for reference in references - if reference.ref_kind == SqlReferenceKind.SOURCE - ], - ) - ) - return cte_sources, None - - -def validate_test_ctes( - *, - test_ctes: CompileSqlTestCtes, - test_file: DiscoveredSqlTestFile, - known_model_names: set[str], - known_seed_names: set[str], - known_source_names: set[str], - known_table_function_names: set[str], - loaded_macros: dict[str, LoadedMacro], - assertion_target_model_names: tuple[str, ...], - target_catalog: _native.SqlTestTargetCatalog | None = None, -) -> None: - """Validate SQL-native test CTE targets; preview reuses one native catalog per compile.""" - - if isinstance(test_ctes.payload, CompileDirectLogicSqlTestCtes): - return - - model_payload: CompileModelSqlTestCtes = test_ctes.payload - catalog: _native.SqlTestTargetCatalog = target_catalog or native_test_target_catalog( - models=known_model_names, - sources=known_source_names, - seeds=known_seed_names, - table_functions=known_table_function_names, - macros=loaded_macros, - ) - error: str | None = catalog.unknown_test_target( - str(test_file.relative_path), - ( - [*model_payload.mock_model_names], - [*model_payload.mock_source_names], - [*model_payload.mock_seed_names], - [*model_payload.mock_table_function_names], - [*model_payload.macro_mocks], - [*model_payload.expected_model_names], - [*assertion_target_model_names], - ), - ) - if error is not None: - raise CompileInputError(error) - return diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py index 4bbb400d85..af41cc3efc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py @@ -15,6 +15,7 @@ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, RETIRED_FACT_CACHE_DIRECTORY_NAME, + RETIRED_REFERENCE_CACHE_DIRECTORY_NAME, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier.constants import ( @@ -70,21 +71,18 @@ def compile_cache_root( ): return None root: Path = compiler_cache_directory(project_dir) - _remove_retired_fact_caches(root=root) + _remove_retired_caches(root=root) return root -def _remove_retired_fact_caches(*, root: Path) -> None: - """Delete the per-file fact caches older releases kept under every engine's compiler root.""" +def _remove_retired_caches(*, root: Path) -> None: + """Delete the fact and reference caches older releases kept under every engine's root.""" for suffix in RETIRED_CACHE_NAMESPACE_SUFFIXES: - retired: Path = ( - root.parent - / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" - / (RETIRED_FACT_CACHE_DIRECTORY_NAME) - ) - if retired.is_dir(): - shutil.rmtree(retired, ignore_errors=True) + for name in (RETIRED_FACT_CACHE_DIRECTORY_NAME, RETIRED_REFERENCE_CACHE_DIRECTORY_NAME): + retired: Path = root.parent / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" / name + if retired.is_dir(): + shutil.rmtree(retired, ignore_errors=True) def build_effective_target_namespace( diff --git a/src/sqlbuild/compiler/compile/_helpers/audit_factories/__init__.py b/src/sqlbuild/compiler/compile/_helpers/audit_factories/__init__.py deleted file mode 100644 index 9cdeaf03fa..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/audit_factories/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Audit-factory compile helpers.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/audit_factories/core.py b/src/sqlbuild/compiler/compile/_helpers/audit_factories/core.py deleted file mode 100644 index beef5a6456..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/audit_factories/core.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Audit-factory model attachment helpers.""" - -from __future__ import annotations - -import re -from pathlib import Path - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.models import DiscoveredAuditFactory, DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SourceLocation - - -def build_audit_factory_orphan_diagnostics( - *, discovered_inputs: DiscoveredProjectInputs -) -> tuple[CompilerDiagnostic, ...]: - """Return warnings for audit factories that no model explicitly attaches.""" - - referenced_names: set[str] = set() - for model_file in discovered_inputs.model_files: - raw_references: object | None = model_file.header_values.get("audit_factories") - if isinstance(raw_references, list): - referenced_names.update(item for item in raw_references if isinstance(item, str)) - return tuple( - CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.WARNING, - code="C216", - message=( - f"Audit factory '{factory.name}' is not referenced by any model and will not " - "produce audits" - ), - resource_type=CompiledResourceType.AUDIT, - resource_name=factory.name, - location=SourceLocation(path=factory.relative_path, line=factory.line, column=1), - ) - for factory in discovered_inputs.audit_factories - if factory.name not in referenced_names - ) - - -def parse_model_header_audit_factories( - *, - raw_audit_factories: object | None, - file_path: Path, - model_name: str, - audit_factories: tuple[DiscoveredAuditFactory, ...], -) -> tuple[SchemaAuditInstance, ...]: - """Resolve explicitly named factories into model audit instances.""" - - if raw_audit_factories is None: - return () - if not isinstance(raw_audit_factories, list): - raise CompileInputError(f"{file_path} model audit_factories must be a list") - factories_by_name: dict[str, DiscoveredAuditFactory] = { - factory.name: factory for factory in audit_factories - } - references: list[str] = [] - for value in raw_audit_factories: - if not isinstance(value, str) or not value.strip(): - raise CompileInputError( - f"{file_path} model audit_factories entries must be bare identifiers" - ) - if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", value): - raise CompileInputError( - f"{file_path} model audit_factories entry '{value}' must be a bare identifier" - ) - if value in references: - raise CompileInputError( - f"model '{model_name}' in {file_path} references audit factory '{value}' more " - "than once" - ) - references.append(value) - - generated: list[SchemaAuditInstance] = [] - for reference in references: - factory: DiscoveredAuditFactory | None = factories_by_name.get(reference) - if factory is None: - available: str = ", ".join(sorted(factories_by_name)) or "none discovered" - raise CompileInputError( - f"model '{model_name}' in {file_path} references unknown audit factory " - f"'{reference}'. Available audit factories: {available}" - ) - for case in factory.cases: - generated.append( - SchemaAuditInstance( - definition_name=case.definition, - name=case.name, - arguments=dict(case.arguments), - severity=case.severity, - run_scope=case.run_scope, - always_run=case.always_run, - description=case.description, - thresholds=case.thresholds, - minimum_samples=case.minimum_samples, - evidence_limit=case.evidence_limit, - location=SourceLocation( - path=factory.relative_path, - line=factory.line, - column=1, - ), - ) - ) - return tuple(generated) - - -def merge_validated_model_audits( - *, - direct_audits: tuple[SchemaAuditInstance, ...], - generated_audits: tuple[SchemaAuditInstance, ...], - model_name: str, - file_path: Path, -) -> tuple[SchemaAuditInstance, ...]: - """Append generated audits while rejecting generated identity collisions.""" - - merged: tuple[SchemaAuditInstance, ...] = (*direct_audits, *generated_audits) - seen_names: set[str] = {audit.name or audit.definition_name for audit in direct_audits} - for audit in generated_audits: - identity_name: str = audit.name or audit.definition_name - if identity_name in seen_names: - raise CompileInputError( - f"model '{model_name}' in {file_path} has duplicate audit case name " - f"'{identity_name}' across direct and factory attachments" - ) - seen_names.add(identity_name) - return merged diff --git a/src/sqlbuild/compiler/compile/_helpers/config/cursor_inputs.py b/src/sqlbuild/compiler/compile/_helpers/config/cursor_inputs.py deleted file mode 100644 index 3434f811d2..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/config/cursor_inputs.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Reject cursor-based incremental models that have no input to bound their cursor window.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile.constants import ( - CURSOR_INPUTS_CONFIG_KEY, - CURSOR_MODEL_WITHOUT_INPUTS_CODE, -) -from sqlbuild.compiler.compile.models import CompileModelInput, CompilerDiagnostic -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.main._model_header_entry_span import get_model_header_entry_span -from sqlbuild.compiler.planner.types import MaterializationType, MicrobatchStrategy -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.spec.contracts.main.get_config_str import get_config_str -from sqlbuild.spec.contracts.models import SourceLocation - -_CURSOR_CONFIG_KEY: str = "cursor" -_RELATION_REFERENCE_KINDS: frozenset[SqlReferenceKind] = frozenset( - { - SqlReferenceKind.REF, - SqlReferenceKind.SOURCE, - SqlReferenceKind.SEED, - SqlReferenceKind.DBT_REF, - } -) -_HELP: str = ( - "read the data through __ref(), __source() or __seed() so its cursor can bound the window, " - "or use materialized table if the model has no upstream data" -) - - -def cursor_model_without_inputs_diagnostics( - *, model_inputs: tuple[CompileModelInput, ...] -) -> tuple[CompilerDiagnostic, ...]: - """Return one error per cursor-based incremental model with no cursor inputs.""" - - return tuple( - _diagnostic(model_input) - for model_input in model_inputs - if _lacks_cursor_inputs(model_input) - ) - - -def _lacks_cursor_inputs(model_input: CompileModelInput) -> bool: - values: dict[str, object] = model_input.config.values - if get_config_str(values=values, key="materialized") != MaterializationType.INCREMENTAL: - return False - if get_config_str(values=values, key=_CURSOR_CONFIG_KEY) is None: - return False - if ( - get_config_str(values=values, key="microbatch_strategy") - == MicrobatchStrategy.ROLLING_WINDOW - ): - return False - if values.get(CURSOR_INPUTS_CONFIG_KEY) is not None: - return False - return not any( - reference.ref_kind in _RELATION_REFERENCE_KINDS for reference in model_input.references - ) - - -def _diagnostic(model_input: CompileModelInput) -> CompilerDiagnostic: - name: str = model_input.model_file.file_path.stem - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=CURSOR_MODEL_WITHOUT_INPUTS_CODE, - message=( - f"incremental model '{name}' reads no inputs, so builds after the first " - "cannot work out their cursor window" - ), - resource_type=CompiledResourceType.MODEL, - resource_name=name, - location=_cursor_location( - path=model_input.model_file.relative_path, - contents=model_input.model_file.contents, - ), - help=_HELP, - ) - - -def _cursor_location(*, path: Path, contents: str) -> SourceLocation: - span: tuple[int, int] | None = get_model_header_entry_span( - contents=contents, key=_CURSOR_CONFIG_KEY - ) - start: int = 0 if span is None else span[0] - line: int = contents.count("\n", 0, start) + 1 - column: int = start - (contents.rfind("\n", 0, start) + 1) + 1 - if span is None: - return SourceLocation(path=path, line=line, column=column) - return SourceLocation( - path=path, - line=line, - column=column, - end_line=line, - end_column=column + span[1] - span[0], - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py b/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py index 620664367a..9978f375c3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/dynamic_columns.py @@ -1,88 +1,3 @@ """Dynamic column-family configuration parsing for model headers.""" from __future__ import annotations - -import re -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def parse_dynamic_column_families( - *, raw_value: object | None, model_name: str, file_path: Path -) -> tuple[SchemaDynamicColumnFamily, ...]: - if raw_value is None: - return () - if not isinstance(raw_value, dict) or not raw_value: - raise CompileInputError( - f"model '{model_name}' dynamic_columns must be a non-empty named block in {file_path}" - ) - allowed_keys: frozenset[str] = frozenset( - {"pivot_column", "value_column", "aggregate", "type", "name_pattern"} - ) - families: list[SchemaDynamicColumnFamily] = [] - for family_name, raw_family in raw_value.items(): - if not isinstance(family_name, str) or not family_name: - raise CompileInputError( - f"model '{model_name}' dynamic column family names must be non-empty strings" - ) - if not isinstance(raw_family, dict): - raise CompileInputError( - f"model '{model_name}' dynamic column family '{family_name}' must be a block" - ) - family_values: dict[str, object] = {str(key): value for key, value in raw_family.items()} - unknown_keys: set[str] = set(family_values) - allowed_keys - if unknown_keys: - raise CompileInputError( - f"model '{model_name}' dynamic column family '{family_name}' has unsupported keys: " - f"{', '.join(sorted(unknown_keys))}" - ) - pivot_column: object = family_values.get("pivot_column") - value_column: object = family_values.get("value_column") - aggregate: object = family_values.get("aggregate") - data_type: object = family_values.get("type") - for key, value in ( - ("pivot_column", pivot_column), - ("value_column", value_column), - ("aggregate", aggregate), - ("type", data_type), - ): - if not isinstance(value, str) or not value.strip(): - raise CompileInputError( - f"model '{model_name}' dynamic column family '{family_name}' requires a " - f"non-empty {key}" - ) - name_pattern: object = family_values.get("name_pattern") - if name_pattern is not None and ( - not isinstance(name_pattern, str) or not name_pattern.strip() - ): - raise CompileInputError( - f"model '{model_name}' dynamic column family '{family_name}' name_pattern must be " - "a non-empty regular expression" - ) - if isinstance(name_pattern, str): - try: - re.compile(name_pattern) - except re.error as error: - raise CompileInputError( - f"model '{model_name}' dynamic column family '{family_name}' has invalid " - f"name_pattern: {error}" - ) from error - families.append( - SchemaDynamicColumnFamily( - name=family_name, - pivot_column=cast(str, pivot_column), - value_column=cast(str, value_column), - aggregate=cast(str, aggregate).upper(), - type=cast(str, data_type), - name_pattern=name_pattern, - ) - ) - if len(families) > 1 and any(family.name_pattern is None for family in families): - raise CompileInputError( - f"model '{model_name}' declares multiple dynamic column families; each family requires " - "name_pattern so runtime columns can be classified unambiguously" - ) - return tuple(families) diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py index d922fda9dd..435a4ef422 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py @@ -1,90 +1,3 @@ """Compile-time validation of model config combinations.""" from __future__ import annotations - -from sqlbuild.compiler.compile.constants import ( - MIGRATE_FROM_CONFIG_KEY, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.planner.types import ( - MaterializationType, -) -from sqlbuild.spec.contracts.main.get_config_str import get_config_str -from sqlbuild.spec.contracts.models import ( - SchemaColumn, -) - -_QUALIFIED_NAME_SEPARATOR: str = "." - -_HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( - {MaterializationType.INCREMENTAL, MaterializationType.SNAPSHOT} -) -_MIGRATABLE_MATERIALIZATIONS: frozenset[str] = frozenset( - { - MaterializationType.TABLE, - MaterializationType.VIEW, - MaterializationType.INCREMENTAL, - MaterializationType.SNAPSHOT, - } -) - - -def validate_column_migration_config( - *, config_values: dict[str, object], model_name: str, columns: tuple[SchemaColumn, ...] -) -> None: - """Validate column-level migrate_from declarations as one rename set.""" - - declared: tuple[SchemaColumn, ...] = tuple( - column for column in columns if column.migrate_from is not None - ) - if not declared: - return - materialized: str | None = get_config_str(values=config_values, key="materialized") - if materialized not in _HISTORY_MATERIALIZATIONS and ( - materialized not in _MIGRATABLE_MATERIALIZATIONS - or config_values.get(MIGRATE_FROM_CONFIG_KEY) is None - ): - raise CompileInputError( - f"model '{model_name}': column '{declared[0].name}' declares migrate_from, which is " - "only valid for incremental and snapshot models, or for table and view models " - "that declare migrate_from themselves (the old name's compatibility view then " - f"presents the column under its old name); '{materialized}' models are rebuilt " - "with their new columns", - help="remove migrate_from from the column, or declare the model's own migrate_from", - ) - sources: dict[str, str] = {} - column: SchemaColumn - for column in declared: - origin: str = (column.migrate_from or "").strip() - if not origin or _QUALIFIED_NAME_SEPARATOR in origin: - raise CompileInputError( - f"model '{model_name}': column '{column.name}' migrate_from must name one " - "column of the model's existing table", - help="write the old column name, for example revenue (migrate_from amount)", - ) - if origin.lower() == column.name.lower(): - raise CompileInputError( - f"model '{model_name}': column '{column.name}' migrate_from cannot name the " - "column itself" - ) - claimed_by: str | None = sources.get(origin.lower()) - if claimed_by is not None: - raise CompileInputError( - f"model '{model_name}': columns '{claimed_by}' and '{column.name}' both " - f"declare migrate_from {origin}; a column can be renamed to only one new name", - help="keep migrate_from on the column that should receive the existing data", - ) - sources[origin.lower()] = column.name - destinations: dict[str, str] = {column.name.lower(): column.name for column in declared} - for column in declared: - chained: str | None = destinations.get((column.migrate_from or "").strip().lower()) - if chained is not None: - raise CompileInputError( - f"model '{model_name}': column '{column.name}' migrates from " - f"'{column.migrate_from}', which itself declares migrate_from; chained or " - "swapped column renames are not supported", - help=( - "a chain or swap cannot be renamed in place safely; give the new column a " - "name no other renamed column uses, or rebuild the model with --full-refresh" - ), - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/config/namespace_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/namespace_validation.py deleted file mode 100644 index 53bf0c2e58..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/config/namespace_validation.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Validate logical namespaces required by preserve target values.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.constants import PRESERVE_TARGET_VALUE -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.spec.contracts.models import TargetConfig - - -def validate_preserved_logical_namespace( - *, - resource_label: str, - logical_database: str | None, - logical_schema: str | None, - target_config: TargetConfig | None, -) -> None: - """Fail when preserve cannot retain a resource-owned logical namespace.""" - - if target_config is None: - return - missing: list[str] = [] - if target_config.database == PRESERVE_TARGET_VALUE and logical_database is None: - missing.append("database") - if target_config.schema == PRESERVE_TARGET_VALUE and logical_schema is None: - missing.append("schema") - if not missing: - return - dimensions: str = " and ".join(missing) - raise CompileInputError( - f"{resource_label} has no logical {dimensions}, but the selected target sets " - f"{dimensions} to 'preserve'", - help=( - f"Set {dimensions} on the resource or its defaults, or configure a literal target " - f"{dimensions}." - ), - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py index ae6080acb7..2242fe26a1 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py @@ -5,18 +5,21 @@ from collections.abc import Callable, Iterator from contextlib import contextmanager from contextvars import ContextVar, Token -from dataclasses import replace from functools import wraps +from typing import cast +import sqlbuild._native as _native from sqlbuild.compiler.compile.classes.collected_compile_diagnostics import ( CollectedCompileDiagnostics, ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( - CompiledProject, CompileProjectInputs, CompilerDiagnostic, ) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView +from sqlbuild.compiler.compiled_project.main.inputs_with import inputs_with +from sqlbuild.compiler.compiled_project.main.project_with import project_with _ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar( "sqlbuild_collected_compile_diagnostics", default=None @@ -35,7 +38,7 @@ def collect_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics]: _ACTIVE_COLLECTOR.reset(token) -def with_collected_compile_diagnostics[**P, R: (CompileProjectInputs, CompiledProject)]( +def with_collected_compile_diagnostics[**P, R: (CompileInputsView, CompiledProjectView)]( build: Callable[P, R], ) -> Callable[P, R]: """Append every diagnostic reported while ``build`` runs to the diagnostics it returns.""" @@ -46,7 +49,14 @@ def collecting(*args: P.args, **kwargs: P.kwargs) -> R: built: R = build(*args, **kwargs) if not violations.diagnostics: return built - return replace(built, diagnostics=(*built.diagnostics, *violations.diagnostics)) + diagnostics: tuple[CompilerDiagnostic, ...] = ( + *built.diagnostics, + *violations.diagnostics, + ) + if isinstance(built, CompileProjectInputs | _native.NativeInputsView): + return cast(R, inputs_with(inputs=built, diagnostics=diagnostics)) + project: CompiledProjectView = cast(CompiledProjectView, built) + return cast(R, project_with(project=project, diagnostics=diagnostics)) return collecting diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py deleted file mode 100644 index ec08682bef..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/descriptions.py +++ /dev/null @@ -1,458 +0,0 @@ -"""Require a non-empty description on every named project resource.""" - -from __future__ import annotations - -import ast -import inspect -from collections.abc import Callable, Iterator -from dataclasses import dataclass -from functools import partial -from pathlib import Path - -from sqlbuild.compiler.compile.constants import MISSING_DESCRIPTION_CODE -from sqlbuild.compiler.compile.models import ( - CompileModelInput, - CompilerDiagnostic, - CompileSeedInput, - CompileSourceInput, - CompileSqlFunctionInput, -) -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, - FunctionLanguage, -) -from sqlbuild.compiler.discovery.constants import PYTHON_UDF_DECORATOR_NAME, YAML_FILE_SUFFIXES -from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line -from sqlbuild.compiler.discovery.models import ( - DiscoveredLoaderFunction, - DiscoveredProjectInputs, - DiscoveredProvider, -) -from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment -from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.snippet_help import snippet_help -from sqlbuild.spec.contracts.models import SourceLocation - -_LINE_COMMENT_START: str = "--" -_BLOCK_COMMENT_START: str = "/*" - - -@dataclass(frozen=True) -class _Resource: - """One named resource and the description it declares, if any.""" - - kind: str - name: str - description: str | None - location: Callable[[], SourceLocation] - help: str - resource_type: CompiledResourceType | None = None - - -@dataclass(frozen=True) -class _PythonNode: - """The shared shape of a decorated Python node.""" - - kind: str - decorator: str - name: str - description: str | None - relative_path: Path - function: Callable[..., object] - - -def missing_description_diagnostics( - *, - discovered_inputs: DiscoveredProjectInputs, - model_inputs: tuple[CompileModelInput, ...], - seed_inputs: tuple[CompileSeedInput, ...], - source_inputs: tuple[CompileSourceInput, ...], - function_inputs: tuple[CompileSqlFunctionInput, ...], -) -> tuple[CompilerDiagnostic, ...]: - """Return one error for each named resource without a non-empty description.""" - - loaders_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in discovered_inputs.loader_functions - } - resources: tuple[_Resource, ...] = ( - *(_model(model_input) for model_input in model_inputs), - *_scenarios(discovered_inputs), - *(_seed(seed_input) for seed_input in seed_inputs), - *( - _source(source_input=source_input, loaders_by_name=loaders_by_name) - for source_input in source_inputs - ), - *(_function(function_input) for function_input in function_inputs), - *_sql_hooks(discovered_inputs), - *( - _python_resource(node) - for node in _python_nodes( - discovered_inputs=discovered_inputs, source_inputs=source_inputs - ) - ), - *(_provider(provider) for provider in discovered_inputs.providers), - ) - return tuple( - _diagnostic(resource) - for resource in resources - if resource.description is None or not resource.description.strip() - ) - - -def _diagnostic(resource: _Resource) -> CompilerDiagnostic: - state: str = ( - "has no description" if resource.description is None else "has an empty description" - ) - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=MISSING_DESCRIPTION_CODE, - message=f"{resource.kind} '{resource.name}' {state}", - resource_type=resource.resource_type, - resource_name=resource.name, - location=resource.location(), - help=resource.help, - ) - - -def _model(model_input: CompileModelInput) -> _Resource: - name: str = model_input.model_file.file_path.stem - path: Path = model_input.model_file.relative_path - contents: str = model_input.model_file.contents - help_text: str = _header_help( - kind="model", - name=name, - statement="MODEL", - entry=f'description "What one row of {name} represents"', - path=path, - ) - schema_entry_description: str | None = ( - model_input.schema_entry.description if model_input.schema_entry is not None else None - ) - return _Resource( - kind="model", - name=name, - description=schema_entry_description, - location=partial(_header_location, path=path, contents=contents), - help=help_text, - resource_type=CompiledResourceType.MODEL, - ) - - -def _scenarios(discovered_inputs: DiscoveredProjectInputs) -> Iterator[_Resource]: - for scenario_file in discovered_inputs.scenario_files: - raw_description: object | None = scenario_file.header_values.get("description") - yield _Resource( - kind="scenario", - name=scenario_file.name, - description=raw_description if isinstance(raw_description, str) else None, - location=partial( - _header_location, - path=scenario_file.relative_path, - contents=scenario_file.contents, - ), - help=_header_help( - kind="scenario", - name=scenario_file.name, - statement="SCENARIO", - entry=f'description "The behaviour {scenario_file.name} proves"', - path=scenario_file.relative_path, - ), - resource_type=CompiledResourceType.SQL_SCENARIO, - ) - - -def _seed(seed_input: CompileSeedInput) -> _Resource: - name: str = seed_input.schema_entry.name - return _Resource( - kind="seed", - name=name, - description=seed_input.schema_entry.description, - location=partial( - _yaml_location, - path=seed_input.schema_file.relative_path, - contents=seed_input.schema_file.contents, - name=name, - ), - help=_yaml_help(kind="seed", name=name, path=seed_input.schema_file.relative_path), - resource_type=CompiledResourceType.SEED, - ) - - -def _source( - *, source_input: CompileSourceInput, loaders_by_name: dict[str, DiscoveredLoaderFunction] -) -> _Resource: - name: str = source_input.source_entry.name - loader: DiscoveredLoaderFunction | None = ( - loaders_by_name.get(source_input.source_entry.loader) - if source_input.source_entry.loader is not None - else None - ) - description: str | None = source_input.source_entry.description - if (description is None or not description.strip()) and loader is not None: - description = loader.description or description - help_text: str = _yaml_help( - kind="source", name=name, path=source_input.source_file.relative_path - ) - if loader is not None and loader.relative_path.suffix not in YAML_FILE_SUFFIXES: - help_text = join_helps( - help_text, - f"or describe its loader '{loader.name}' in {loader.relative_path.as_posix()} " - f'with a docstring or @loader(description="What one row of {name} represents")', - ) - return _Resource( - kind="source", - name=name, - description=description, - location=partial( - _yaml_location, - path=source_input.source_file.relative_path, - contents=source_input.source_file.contents, - name=name, - ), - help=help_text, - resource_type=CompiledResourceType.SOURCE, - ) - - -def _function(function_input: CompileSqlFunctionInput) -> _Resource: - name: str = function_input.name - path: Path = function_input.function_file.relative_path - resource_type: CompiledResourceType = ( - CompiledResourceType.TABLE_FN if function_input.return_columns else CompiledResourceType.UDF - ) - location: Callable[[], SourceLocation] - help_text: str - if function_input.language == FunctionLanguage.PYTHON: - location = partial( - _decorator_location, - path=path, - contents=function_input.function_file.contents, - decorator=PYTHON_UDF_DECORATOR_NAME, - ) - help_text = snippet_help( - purpose=f"to describe function '{name}'", - target=f"give its function in {path.as_posix()} a docstring or add this to @udf", - lines=(f'@udf(description="What {name} returns", ...)',), - ) - else: - location = partial( - _header_location, path=path, contents=function_input.function_file.contents - ) - help_text = _header_help( - kind="function", - name=name, - statement="FUNCTION", - entry=f'description "What {name} returns"', - path=path, - ) - return _Resource( - kind="function", - name=name, - description=function_input.description, - location=location, - help=help_text, - resource_type=resource_type, - ) - - -def _sql_hooks(discovered_inputs: DiscoveredProjectInputs) -> Iterator[_Resource]: - for hook_file in discovered_inputs.sql_hook_files: - yield _Resource( - kind="hook", - name=hook_file.name, - description=hook_file.description, - location=partial( - _header_location, path=hook_file.relative_path, contents=hook_file.contents - ), - help=_header_help( - kind="hook", - name=hook_file.name, - statement="HOOK", - entry=f'description "What {hook_file.name} does"', - path=hook_file.relative_path, - description_only=True, - ), - ) - - -def _python_nodes( - *, discovered_inputs: DiscoveredProjectInputs, source_inputs: tuple[CompileSourceInput, ...] -) -> Iterator[_PythonNode]: - source_owned_loaders: frozenset[str] = frozenset( - source_input.source_entry.loader - for source_input in source_inputs - if source_input.source_entry.loader is not None - ) - source_paths: frozenset[Path] = frozenset( - source_file.relative_path for source_file in discovered_inputs.source_files - ) - for loader in discovered_inputs.loader_functions: - if loader.name in source_owned_loaders or loader.relative_path in source_paths: - continue - yield _PythonNode( - kind="loader", - decorator="loader", - name=loader.name, - description=loader.description, - relative_path=loader.relative_path, - function=loader.function, - ) - for kind, functions in ( - ("task", discovered_inputs.task_functions), - ("asset", discovered_inputs.asset_functions), - ("check", discovered_inputs.check_functions), - ("hook", discovered_inputs.hook_functions), - ): - for function in functions: - yield _PythonNode( - kind=kind, - decorator=kind, - name=function.name, - description=function.description, - relative_path=function.relative_path, - function=function.function, - ) - - -def _python_resource(node: _PythonNode) -> _Resource: - return _Resource( - kind=node.kind, - name=node.name, - description=node.description, - location=partial(_function_location, path=node.relative_path, function=node.function), - help=snippet_help( - purpose=f"to describe {node.kind} '{node.name}'", - target=( - f"give its function in {node.relative_path.as_posix()} a docstring " - f"or pass a description to @{node.decorator}" - ), - lines=(f'@{node.decorator}(description="What {node.name} does", ...)',), - ), - ) - - -def _provider(provider: DiscoveredProvider) -> _Resource: - class_name: str = provider.provider_class.__name__ - raw_doc: object | None = provider.provider_class.__dict__.get("__doc__") - return _Resource( - kind="provider", - name=provider.name, - description=inspect.cleandoc(raw_doc) if isinstance(raw_doc, str) else None, - location=partial( - _class_location, path=provider.relative_path, provider_class=provider.provider_class - ), - help=snippet_help( - purpose=f"to describe provider '{provider.name}'", - target=f"add a class docstring in {provider.relative_path.as_posix()}", - lines=( - f"class {class_name}(Provider):", - f' """What {provider.name} provides."""', - ), - ), - ) - - -def _header_help( - *, - kind: str, - name: str, - statement: str, - entry: str, - path: Path, - description_only: bool = False, -) -> str: - lines: tuple[str, ...] = ( - (f"{statement} ({entry});",) - if description_only - else (f"{statement} (", f" {entry},", " ...", ");") - ) - return snippet_help( - purpose=f"to describe {kind} '{name}'", - target=f"add this to the {statement} header in {path.as_posix()}", - lines=lines, - ) - - -def _yaml_help(*, kind: str, name: str, path: Path) -> str: - return snippet_help( - purpose=f"to describe {kind} '{name}'", - target=f"add `description` to its entry in {path.as_posix()}", - lines=(f"- name: {name}", f" description: What one row of {name} represents"), - ) - - -def _header_location(*, path: Path, contents: str) -> SourceLocation: - """Locate the statement header: the first code after leading whitespace and comments.""" - - position: int = 0 - while True: - while position < len(contents) and contents[position].isspace(): - position += 1 - if contents.startswith(_LINE_COMMENT_START, position): - position = skip_line_comment(sql=contents, start=position) - elif contents.startswith(_BLOCK_COMMENT_START, position): - position = skip_block_comment(sql=contents, start=position) - else: - return _offset_location(path=path, contents=contents, offset=position) - - -def _function_location(*, path: Path, function: Callable[..., object]) -> SourceLocation: - return SourceLocation(path=path, line=_function_line(function), column=1) - - -def _class_location(*, path: Path, provider_class: type[object]) -> SourceLocation: - return SourceLocation(path=path, line=_class_line(provider_class), column=1) - - -def _yaml_location(*, path: Path, contents: str, name: str) -> SourceLocation: - line: int | None = yaml_entry_line(contents=contents, name=name) - return SourceLocation(path=path, line=line or 1, column=1) - - -def _decorator_location(*, path: Path, contents: str, decorator: str) -> SourceLocation: - """Locate the `@decorator` line of the decorated function, ignoring docstrings and comments.""" - - try: - module: ast.Module = ast.parse(contents) - except SyntaxError: - return SourceLocation(path=path, line=1, column=1) - for node in ast.walk(module): - if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): - for expression in node.decorator_list: - if _decorator_name(expression) == decorator: - return SourceLocation( - path=path, line=expression.lineno, column=expression.col_offset - ) - return SourceLocation(path=path, line=1, column=1) - - -def _decorator_name(expression: ast.expr) -> str | None: - target: ast.expr = expression.func if isinstance(expression, ast.Call) else expression - if isinstance(target, ast.Name): - return target.id - if isinstance(target, ast.Attribute): - return target.attr - return None - - -def _offset_location(*, path: Path, contents: str, offset: int) -> SourceLocation: - line: int = contents.count("\n", 0, offset) + 1 - column: int = offset - (contents.rfind("\n", 0, offset) + 1) + 1 - return SourceLocation(path=path, line=line, column=column) - - -def _function_line(function: Callable[..., object]) -> int: - code: object | None = getattr(inspect.unwrap(function), "__code__", None) - line: object | None = getattr(code, "co_firstlineno", None) - return line if isinstance(line, int) else 1 - - -def _class_line(provider_class: type[object]) -> int: - try: - return inspect.getsourcelines(provider_class)[1] - except (OSError, TypeError): - return 1 diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py deleted file mode 100644 index 86257468d3..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Column suggestions and unknown-column parsing for resource SQL diagnostics.""" - -from __future__ import annotations - -import re -from typing import Any - -_MIN_ABBREVIATION_LENGTH: int = 3 -_MAX_EDIT_DISTANCE: int = 2 -_MISSING: re.Pattern[str] = re.compile(r"Unknown column '([^']+)'(?: in table '([^']+)')?") - - -def column_distance(*, left: str, right: str) -> int: - """Levenshtein distance, also used to order schema evidence.""" - left, right = left.casefold(), right.casefold() - row: list[int] = list(range(len(right) + 1)) - for i, a in enumerate(left, 1): - following: list[int] = [i] - for j, b in enumerate(right, 1): - following.append(min(following[-1] + 1, row[j] + 1, row[j - 1] + (a != b))) - row = following - return row[-1] - - -def ordered_columns(*, name: str, columns: dict[str, str]) -> list[str]: - return sorted( - columns, - key=lambda value: ( - column_distance(left=name, right=value) / max(len(name), len(value), 1), - value, - ), - ) - - -def closest_column(*, name: str, columns: dict[str, str]) -> str | None: - candidates: list[str] = ordered_columns(name=name, columns=columns) - for candidate in candidates: - distance: int = column_distance(left=name, right=candidate) - remaining: Any = iter(candidate.casefold()) - abbreviation: bool = ( - len(name) >= _MIN_ABBREVIATION_LENGTH - and candidate[:1].casefold() == name[:1].casefold() - and candidate[-1:].casefold() == name[-1:].casefold() - and all(character in remaining for character in name.casefold()) - ) - if distance <= _MAX_EDIT_DISTANCE or abbreviation: - return candidate - return None - - -def missing_column(message: str) -> tuple[str, str | None] | None: - match: re.Match[str] | None = _MISSING.search(message) - return (match.group(1), match.group(2)) if match else None diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py index 212d9a5dd2..d62407d202 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -5,21 +5,21 @@ from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.analysis_session.types import DiagnosticRow +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( complete_native_semantic_diagnostics, ) -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic def complete_semantic_diagnostics( *, - project: CompiledProject, + project: CompiledProjectView, profile: ExpressionInferenceProfile, - binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + binding_results: dict[str, list[DiagnosticRow]], resource_sql_analysis: bool = True, native_session: Any | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """Recover type facts before metadata checks, then explain retained root diagnostics.""" return complete_native_semantic_diagnostics( diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py index a038120917..9712e43569 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py @@ -1,523 +1,165 @@ -"""Run the model compile-time SQL checks on audits, SQL tests and SQL hooks.""" +"""Hand audit, SQL test and SQL hook SQL to the native resource SQL checks.""" from __future__ import annotations -import re -from dataclasses import dataclass from pathlib import Path +from typing import Any, cast +import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.binding_positions import ( - get_authored_binding_location, -) -from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( - known_declared_types, - known_function_names, -) from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import binding_relation_names -from sqlbuild.compiler.compile._helpers.diagnostics.resource_sql_help import resource_sql_help -from sqlbuild.compiler.compile._helpers.diagnostics.sql_analysis_opt_outs import ( - unneeded_opt_out_diagnostic, -) -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsics_analysis_sql, -) -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledProject, - CompiledSqlExpansion, - CompilerDiagnostic, - CompileSqlReference, -) +from sqlbuild.compiler.compile.models import CompilerDiagnostic, CompileSqlReference from sqlbuild.compiler.compile.types import ( + CompiledAuditView, + CompiledModelView, + CompiledProjectView, CompiledResourceType, DiagnosticPhase, DiagnosticSeverity, ) from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY -from sqlbuild.compiler.sql_analysis.constants import BINDING_UNKNOWN_TABLE_INTERNAL_CODE -from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql -from sqlbuild.compiler.sql_analysis.main._resolve_binding_column import resolve_binding_column -from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations -from sqlbuild.compiler.sql_analysis.models import ( - SqlBindingDiagnostic, - SqlBindingResult, - SqlSchemaValidationRequest, -) from sqlbuild.spec.contracts.models import SourceLocation _MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") -_QUOTED_NAME_PATTERN: re.Pattern[str] = re.compile(r"'([^']+)'") -_REFERENCE_CALL_PATTERN: re.Pattern[str] = re.compile( - r"__(?:ref|source|seed)\(\s*['\"]([^'\"]+)['\"]\s*\)" -) -_EMPTY_FIXTURE_CTE_PATTERN: re.Pattern[str] = re.compile( - r"(?P\b__(?:ref|source|seed|expected)__(?P\w+)\s+AS\s*\(\s*SELECT\s+\*\s+FROM\s+)" - r"__empty_fixture\s*\(\s*\)", - flags=re.IGNORECASE, -) -_EMPTY_FIXTURE_CALL_PATTERN: re.Pattern[str] = re.compile( - r"\b__empty_fixture\s*\(\s*\)", flags=re.IGNORECASE -) -_OPEN_EMPTY_FIXTURE_RELATION: str = "__sqlbuild_empty_fixture" _BUILT_IN_AUDIT_ROOT: str = "" -_COLUMN_ONLY_BUILT_IN_AUDITS: frozenset[str] = frozenset({"not_null", "unique"}) -_UNKNOWN_TABLE_MESSAGE_PREFIX: str = "Unknown table " -_SYNTAX_ERROR_CODE: str = "P001" -_RESOURCE_KIND_LABELS: dict[CompiledResourceType, str] = { - CompiledResourceType.AUDIT: "audit", - CompiledResourceType.SQL_TEST: "SQL test", - CompiledResourceType.MODEL: "model", +_RESOURCE_TYPES: dict[str, CompiledResourceType] = { + "audit": CompiledResourceType.AUDIT, + "sql_test": CompiledResourceType.SQL_TEST, + "model": CompiledResourceType.MODEL, } - -@dataclass(frozen=True) -class _ResourceSql: - """One authored SQL body owned by a non-model resource.""" - - resource_type: CompiledResourceType - resource_name: str - path: Path - contents: str - authored_body: str - sql: str - relations: frozenset[str] - label: str | None = None - audit_definition: str | None = None - opt_out: SourceLocation | None = None - - -@dataclass(frozen=True) -class _HelpContext: - shapes: dict[str, dict[str, str]] - dialect: str | None +type _Span = tuple[int, int, int | None, int | None] +type _Row = tuple[str, str, bool, str, str, str, _Span, str | None, str | None] def get_resource_sql_diagnostics( *, - project: CompiledProject, + project: CompiledProjectView, shapes: dict[str, dict[str, str]], profile: ExpressionInferenceProfile, ) -> tuple[CompilerDiagnostic, ...]: """Bind audits, SQL tests and SQL hooks against the same closed shapes as models.""" - resources: tuple[_ResourceSql, ...] = tuple( - resource - for resource in dict.fromkeys( + request: Any = ( + profile.sql_analysis_dialect, + profile.quoted_identifiers_ignore_case, + project.settings.require_sql_analysis, + [(name, list(columns.items())) for name, columns in shapes.items()], + [_audit_row(audit) for audit in project.audits], + [ ( - *_audit_resources(project=project, shapes=shapes, profile=profile), - *_sql_test_resources(project=project), - *_hook_resources(project=project), - ) - ) - if resource.opt_out is None or project.settings.require_sql_analysis - ) - if not resources: - return () - dialect: str | None = profile.sql_analysis_dialect - normalized: dict[str, str] = {} - for resource in resources: - if resource.sql not in normalized: - normalized[resource.sql] = normalize_analysis_sql( - sql=cursor_intrinsics_analysis_sql(sql=resource.sql, cursor_type=None), - dialect=dialect, + test.name, + list(test.expected_model_names), + str(test.test_file.relative_path), + test.test_file.contents, + test.sql_body, + test.test_block.sql_body, + test.test_block.header_values.get(SQL_ANALYSIS_CONFIG_KEY) is False, ) - cleaned: tuple[str, ...] = tuple(normalized[resource.sql] for resource in resources) - known_functions: tuple[str, ...] = known_function_names(project.functions) - known_types: tuple[str, ...] = known_declared_types( - functions=project.functions, column_types=shapes - ) - requests: dict[tuple[str, frozenset[str]], SqlSchemaValidationRequest] = {} - for resource, cleaned_sql in zip(resources, cleaned, strict=True): - _ = requests.setdefault( - (cleaned_sql, resource.relations), - SqlSchemaValidationRequest( - sql=cleaned_sql, - dialect=dialect, - schema=_relation_schema(relations=resource.relations, shapes=shapes), - known_functions=known_functions, - known_types=known_types, - quoted_identifiers_ignore_case=profile.quoted_identifiers_ignore_case, - catalog=project.binding_catalog, - ), - ) - unique_results: dict[tuple[str, frozenset[str]], SqlBindingResult] = dict( - zip(requests, get_schema_validations(requests=tuple(requests.values())), strict=True) + for test in project.sql_tests + ], + [_model_row(model) for model in project.models], + [ + (str(hook_file.relative_path), hook_file.contents) + for hook_file in project.sql_hook_files + ], ) - results: tuple[SqlBindingResult, ...] = tuple( - unique_results[(cleaned_sql, resource.relations)] - for resource, cleaned_sql in zip(resources, cleaned, strict=True) + rows: list[_Row] = cast( + list[_Row], + _native.resource_sql_diagnostic_rows(cast(Any, project.binding_catalog).native, request), ) - diagnostics: dict[tuple[object, ...], CompilerDiagnostic] = {} - for resource, cleaned_sql, result in zip(resources, cleaned, results, strict=True): - found: dict[tuple[object, ...], CompilerDiagnostic] = _resource_diagnostics( - resource=resource, - cleaned_sql=cleaned_sql, - result=result, - context=_HelpContext(shapes=shapes, dialect=dialect), - ) - if resource.opt_out is None: - for key, diagnostic in found.items(): - diagnostics.setdefault(key, diagnostic) - elif not any(item.code == _SYNTAX_ERROR_CODE for item in found.values()): - diagnostics.setdefault( - (resource.opt_out.path, resource.opt_out.line, resource.opt_out.column), - unneeded_opt_out_diagnostic( - resource_type=resource.resource_type, - kind=_RESOURCE_KIND_LABELS[resource.resource_type], - name=resource.resource_name, - location=resource.opt_out, - hidden=tuple(found.values()), - ), - ) - return tuple(diagnostics.values()) + paths: dict[str, Path] = {str(path): path for path in _paths(project)} + return tuple(_diagnostic(row=row, paths=paths) for row in rows) -def _resource_diagnostics( - *, - resource: _ResourceSql, - cleaned_sql: str, - result: SqlBindingResult, - context: _HelpContext, -) -> dict[tuple[object, ...], CompilerDiagnostic]: - found: dict[tuple[object, ...], CompilerDiagnostic] = {} - for index, native in enumerate(result.diagnostics): - diagnostic: SqlBindingDiagnostic | None = _reported_diagnostic( - resource=resource, diagnostic=native - ) - if diagnostic is None: - continue - location: SourceLocation = _location( - resource=resource, cleaned_sql=cleaned_sql, diagnostic=diagnostic - ) - key: tuple[object, ...] = ( - resource.path, - diagnostic.code, - diagnostic.message, - location.line, - location.column, - index if diagnostic.start is None else -1, - ) - found.setdefault( - key, - CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity(diagnostic.severity), - code=diagnostic.code, - message=diagnostic.message - if resource.label is None or diagnostic.code == _SYNTAX_ERROR_CODE - else f"{resource.label}: {diagnostic.message}", - resource_type=resource.resource_type, - resource_name=resource.resource_name, - location=location, - help=resource_sql_help( - resource_type=resource.resource_type, - diagnostic=diagnostic, - authored_text=_text_from(resource=resource, location=location), - authored_line=_line_at(resource=resource, location=location), - column=location.column, - authored_body=resource.authored_body, - audit_definition=resource.audit_definition, - shapes=context.shapes, - relations=resource.relations, - dialect=context.dialect, - ), - ), - ) - return found - - -def _relation_schema( - *, relations: frozenset[str], shapes: dict[str, dict[str, str]] -) -> dict[str, dict[str, str]]: - """Return closed shapes for referenced relations; an empty shape stays open.""" +def _paths(project: CompiledProjectView) -> list[Path]: + """Every path a resource SQL diagnostic can name, so rows map back to the authored `Path`.""" - return {name: shapes.get(name) or {} for name in relations} + paths: list[Path] = [audit.audit_file.relative_path for audit in project.audits] + paths.extend(test.test_file.relative_path for test in project.sql_tests) + paths.extend(model.relative_path for model in project.models) + paths.extend(hook_file.relative_path for hook_file in project.sql_hook_files) + return paths -def _reported_diagnostic( - *, resource: _ResourceSql, diagnostic: SqlBindingDiagnostic -) -> SqlBindingDiagnostic | None: - """Drop open-relation lookups and turn parser rejections into syntax errors.""" - - if diagnostic.code != BINDING_UNKNOWN_TABLE_INTERNAL_CODE: - return diagnostic - if diagnostic.message.startswith(_UNKNOWN_TABLE_MESSAGE_PREFIX): - return None - return SqlBindingDiagnostic( - code=_SYNTAX_ERROR_CODE, - message=( - f"SQL syntax error in {_RESOURCE_KIND_LABELS[resource.resource_type]} " - f"'{resource.resource_name}': {diagnostic.message}" - ), - line=diagnostic.line, - column=diagnostic.column, - start=diagnostic.start, - end=diagnostic.end, - ) - - -def _audit_resources( - *, - project: CompiledProject, - shapes: dict[str, dict[str, str]], - profile: ExpressionInferenceProfile, -) -> tuple[_ResourceSql, ...]: - analysis_disabled: frozenset[str] = _analysis_disabled_models(project) - resources: list[_ResourceSql] = [] - for audit in project.audits: - if audit.attached_target_name in analysis_disabled or _binds_cleanly( - audit=audit, shapes=shapes, profile=profile - ): - continue - relations: frozenset[str] = _relations(audit.references) - authored_bodies: tuple[str | None, ...] = ( - audit.audit_block.sql_body, - audit.audit_block.measure_sql, - audit.audit_block.evidence_sql, - ) - for sql, authored in zip( - (audit.sql_body, audit.measure_sql, audit.evidence_sql), authored_bodies, strict=True - ): - if not sql or not sql.strip(): - continue - body: str = authored or sql - resources.append( - _ResourceSql( - resource_type=CompiledResourceType.AUDIT, - resource_name=audit.name, - path=audit.audit_file.relative_path, - contents=audit.audit_file.contents, - authored_body=body, - sql=sql, - relations=relations, - label=_audit_label(audit), - audit_definition=audit.definition_name, - opt_out=_header_opt_out( - header_values=audit.audit_block.header_values, - path=audit.audit_file.relative_path, - contents=audit.audit_file.contents, - body=body, - ), - ) +def _relations(references: tuple[object, ...]) -> list[str]: + return sorted( + binding_relation_names( + tuple( + reference for reference in references if isinstance(reference, CompileSqlReference) ) - return tuple(resources) - - -def _binds_cleanly( - *, audit: CompiledAudit, shapes: dict[str, dict[str, str]], profile: ExpressionInferenceProfile -) -> bool: - """A built-in single-column audit on a column its closed target shape contains.""" - - if ( - audit.definition_name not in _COLUMN_ONLY_BUILT_IN_AUDITS - or audit.audit_file.file_path.parts[:1] != (_BUILT_IN_AUDIT_ROOT,) - or audit.attached_target_name is None - or audit.attached_column_name is None - ): - return False - shape: dict[str, str] = shapes.get(audit.attached_target_name) or {} - return bool(shape) and ( - resolve_binding_column( - name=audit.attached_column_name, - columns=shape, - dialect=profile.sql_analysis_dialect, - ignore_quoted_case=profile.quoted_identifiers_ignore_case, ) - is not None ) -def _audit_label(audit: CompiledAudit) -> str: - label: str = f"audit '{audit.definition_name}'" - if audit.attached_target_name is not None: - label += f" on {audit.attached_target_kind} '{audit.attached_target_name}'" - if audit.attached_column_name is not None: - label += f" column '{audit.attached_column_name}'" - return label - - -def _sql_test_resources(*, project: CompiledProject) -> tuple[_ResourceSql, ...]: - analysis_disabled: frozenset[str] = _analysis_disabled_models(project) - resources: list[_ResourceSql] = [] - prepared: dict[str, tuple[str, frozenset[str]]] = {} - for test in project.sql_tests: - if not test.sql_body.strip() or analysis_disabled.intersection(test.expected_model_names): - continue - if test.sql_body not in prepared: - prepared[test.sql_body] = _prepared_test_sql(test.sql_body) - sql, relations = prepared[test.sql_body] - resources.append( - _ResourceSql( - resource_type=CompiledResourceType.SQL_TEST, - resource_name=test.name, - path=test.test_file.relative_path, - contents=test.test_file.contents, - authored_body=test.test_block.sql_body, - sql=sql, - label=f"SQL test '{test.name}'", - opt_out=_header_opt_out( - header_values=test.test_block.header_values, - path=test.test_file.relative_path, - contents=test.test_file.contents, - body=test.test_block.sql_body, - ), - relations=relations, - ) - ) - return tuple(resources) - - -def _prepared_test_sql(sql_body: str) -> tuple[str, frozenset[str]]: - """Test SQL with empty fixtures opened, and the relations it reads.""" - - sql: str = _EMPTY_FIXTURE_CTE_PATTERN.sub( - lambda match: match.group("prefix") + match.group("name"), sql_body +def _audit_row(audit: CompiledAuditView) -> tuple[object, ...]: + target: tuple[str, str] | None = ( + None + if audit.attached_target_name is None + else (str(audit.attached_target_kind), audit.attached_target_name) ) - return _EMPTY_FIXTURE_CALL_PATTERN.sub(_OPEN_EMPTY_FIXTURE_RELATION, sql), frozenset( - ( - *_REFERENCE_CALL_PATTERN.findall(sql_body), - *(match.group("name") for match in _EMPTY_FIXTURE_CTE_PATTERN.finditer(sql_body)), - ) + return ( + audit.name, + audit.definition_name, + target, + audit.attached_column_name, + audit.audit_file.file_path.parts[:1] == (_BUILT_IN_AUDIT_ROOT,), + str(audit.audit_file.relative_path), + audit.audit_file.contents, + [ + (audit.sql_body, audit.audit_block.sql_body), + (audit.measure_sql, audit.audit_block.measure_sql), + (audit.evidence_sql, audit.audit_block.evidence_sql), + ], + audit.audit_block.header_values.get(SQL_ANALYSIS_CONFIG_KEY) is False, + _relations(audit.references), ) -def _hook_resources(*, project: CompiledProject) -> tuple[_ResourceSql, ...]: - hook_files: dict[Path, str] = { - hook_file.relative_path: hook_file.contents for hook_file in project.sql_hook_files - } - resources: list[_ResourceSql] = [] - for model in project.models: - if model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is False: +def _model_row(model: CompiledModelView) -> tuple[object, ...]: + hooks: list[tuple[object, ...]] = [] + for hook_key in _MODEL_HOOK_KEYS: + entries: object = model.config.values.get(hook_key) + if not isinstance(entries, list | tuple): continue - for hook_key in _MODEL_HOOK_KEYS: - entries: object = model.config.values.get(hook_key) - if not isinstance(entries, list | tuple): + for hook_index, entry in enumerate(cast(list[object] | tuple[object, ...], entries)): + statement: object = getattr(entry, "statement", None) + if not isinstance(statement, str): continue - for hook_index, entry in enumerate(entries): - statement: object = getattr(entry, "statement", None) - if not isinstance(statement, str) or not statement.strip(): - continue - hook_path: Path | None = getattr(entry, "relative_path", None) - definition: object = getattr(entry, "definition_sql", None) - named: bool = hook_path is not None and hook_path in hook_files - resources.append( - _ResourceSql( - resource_type=CompiledResourceType.MODEL, - resource_name=model.name, - path=hook_path if named and hook_path is not None else model.relative_path, - contents=hook_files[hook_path] - if named and hook_path is not None - else model.authored_sql, - authored_body=definition - if named and isinstance(definition, str) - else statement, - sql=statement, - relations=_relations(tuple(getattr(entry, "reads", ()) or ())) - | frozenset(_REFERENCE_CALL_PATTERN.findall(statement)), - label=f"{hook_key}[{hook_index}] SQL hook", - ) + hook_path: Path | None = getattr(entry, "relative_path", None) + definition: object = getattr(entry, "definition_sql", None) + hooks.append( + ( + hook_key, + hook_index, + statement, + None if hook_path is None else str(hook_path), + definition if isinstance(definition, str) else None, + _relations(tuple(getattr(entry, "reads", ()) or ())), ) - return tuple(resources) - - -def _analysis_disabled_models(project: CompiledProject) -> frozenset[str]: - return frozenset( - model.name - for model in project.models - if model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is False - ) - - -def _header_opt_out( - *, header_values: dict[str, object], path: Path, contents: str, body: str -) -> SourceLocation | None: - """Where a TEST or AUDIT header turns SQL analysis off, if it does.""" - - if header_values.get(SQL_ANALYSIS_CONFIG_KEY) is not False: - return None - body_offset: int = contents.find(body) - key_offset: int = contents.rfind( - SQL_ANALYSIS_CONFIG_KEY, 0, body_offset if body_offset >= 0 else len(contents) - ) - return _offset_location(path=path, text=contents, offset=max(key_offset, 0)) - - -def _relations(references: tuple[CompileSqlReference, ...]) -> frozenset[str]: - return binding_relation_names( - tuple(reference for reference in references if isinstance(reference, CompileSqlReference)) + ) + return ( + model.name, + str(model.relative_path), + model.authored_sql, + model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is False, + hooks, ) -def _location( - *, resource: _ResourceSql, cleaned_sql: str, diagnostic: SqlBindingDiagnostic -) -> SourceLocation: - """Point at the offending authored text, falling back to the body start.""" - - body_offset: int = max(resource.contents.find(resource.authored_body), 0) - if diagnostic.start is None: - name_offset: int | None = _quoted_name_offset( - text=resource.contents, start=body_offset, message=diagnostic.message - ) - if name_offset is not None: - return _offset_location(path=resource.path, text=resource.contents, offset=name_offset) - location: SourceLocation | None = get_authored_binding_location( - path=resource.path, - authored_sql=resource.contents, - authored_query_sql=resource.authored_body, - cleaned_sql=cleaned_sql, - diagnostic=diagnostic, - expansion=CompiledSqlExpansion( - authored_sql=resource.authored_body, expanded_sql=resource.sql, passes=() +def _diagnostic(*, row: _Row, paths: dict[str, Path]) -> CompilerDiagnostic: + kind, name, is_error, code, message, path, span, note, help_text = row + line, column, end_line, end_column = span + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR if is_error else DiagnosticSeverity.WARNING, + code=code, + message=message, + resource_type=_RESOURCE_TYPES[kind], + resource_name=name, + location=SourceLocation( + path=paths[path], line=line, column=column, end_line=end_line, end_column=end_column ), + notes=() if note is None else (note,), + help=help_text, ) - if location is not None: - return location - return _offset_location(path=resource.path, text=resource.contents, offset=body_offset) - - -def _quoted_name_offset(*, text: str, start: int, message: str) -> int | None: - for name in _QUOTED_NAME_PATTERN.findall(message): - identifier: str = name.rsplit(".", 1)[-1] - match: re.Match[str] | None = re.search( - rf"(? str: - """Authored text of a diagnostic span (or from its start), for help that quotes the source.""" - - if location.path != resource.path: - return "" - start: int | None = _offset(text=resource.contents, line=location.line, column=location.column) - if start is None: - return "" - end: int | None = ( - _offset(text=resource.contents, line=location.end_line, column=location.end_column) - if location.end_line is not None and location.end_column is not None - else None - ) - return resource.contents[start : end if end is not None and end > start else None] - - -def _line_at(*, resource: _ResourceSql, location: SourceLocation) -> str: - lines: list[str] = resource.contents.splitlines() - if location.path != resource.path or not 1 <= location.line <= len(lines): - return "" - return lines[location.line - 1] - - -def _offset(*, text: str, line: int, column: int) -> int | None: - lines: list[str] = text.splitlines(keepends=True) - if not 1 <= line <= len(lines): - return None - return sum(len(value) for value in lines[: line - 1]) + max(column - 1, 0) - - -def _offset_location(*, path: Path, text: str, offset: int) -> SourceLocation: - line: int = text.count("\n", 0, offset) + 1 - column: int = offset - (text.rfind("\n", 0, offset) + 1) + 1 - return SourceLocation(path=path, line=line, column=column) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql_help.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql_help.py deleted file mode 100644 index e9f3c5f3ff..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql_help.py +++ /dev/null @@ -1,232 +0,0 @@ -"""Concrete remediation for compile checks on SQL tests and audits.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.compile._helpers.diagnostics.details import closest_column, missing_column -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic - -_SET_OPERATION_CODE: str = "B215" -_COMPARISON_CODE: str = "B217" -_UNKNOWN_COLUMN_CODE: str = "B002" -_SNOWFLAKE: str = "snowflake" -_SET_OPERATION_TYPES: re.Pattern[str] = re.compile( - r"column (?P\d+).*accumulated type (?P[\w()]+), next type (?P[\w()]+)", - re.IGNORECASE, -) -_COMPARISON_TYPES: re.Pattern[str] = re.compile( - r"between (?P[\w()]+) and (?P[\w()]+)", re.IGNORECASE -) -_ALIAS_NAME: re.Pattern[str] = re.compile( - r"\s+AS\s+\"?(?P[A-Za-z_][\w$]*)\"?,?$", re.IGNORECASE -) -_ALIAS_SUFFIX: re.Pattern[str] = re.compile(r"\s+AS\s+\"?[A-Za-z_][\w$]*\"?,?$", re.IGNORECASE) -_CAST: re.Pattern[str] = re.compile(r"CAST\((?P.+)\s+AS\s+[\w(), ]+\)", re.IGNORECASE) -_RELATION_PARAMETER: str = "@relation" -_LISTED_COLUMNS: int = 10 -_SINGLE_OPERAND: re.Pattern[str] = re.compile( - r"'(?:[^']|'')*'|[A-Za-z_][\w$]*(?:\.[A-Za-z_][\w$]*)?" -) -_WORD_CHARACTER: re.Pattern[str] = re.compile(r"[\w$]") -_PARAMETER: re.Pattern[str] = re.compile(r"@[A-Za-z_]\w*") -_TEXT_TYPES: tuple[str, ...] = ("VARCHAR", "STRING", "TEXT", "CHAR") -_TEMPORAL_TYPES: tuple[str, ...] = ("TIMESTAMP", "DATE") -_SET_OPERATION_KEYWORDS: frozenset[str] = frozenset( - {"UNION", "ALL", "EXCEPT", "INTERSECT", "MINUS", "SELECT", "FROM", "DISTINCT", "BY", "NAME"} -) - - -def resource_sql_help( - *, - resource_type: CompiledResourceType, - diagnostic: SqlBindingDiagnostic, - authored_text: str, - authored_line: str, - column: int, - authored_body: str, - audit_definition: str | None, - shapes: dict[str, dict[str, str]], - relations: frozenset[str], - dialect: str | None, -) -> str | None: - """Return a concrete fix quoting the authored span, or None to keep the generic help.""" - - if diagnostic.code == _SET_OPERATION_CODE: - return _set_operation_help( - message=diagnostic.message, - authored_text=authored_text, - authored_line=authored_line, - span_column=column, - sql_test=resource_type == CompiledResourceType.SQL_TEST, - relation_shapes={name: shapes.get(name) or {} for name in relations}, - ) - if diagnostic.code == _COMPARISON_CODE and resource_type == CompiledResourceType.AUDIT: - return _audit_comparison_help( - message=diagnostic.message, - authored_line=authored_line, - column=column, - authored_body=authored_body, - audit_definition=audit_definition, - dialect=dialect, - ) - if diagnostic.code == _UNKNOWN_COLUMN_CODE and resource_type == CompiledResourceType.SQL_TEST: - return _unknown_column_help(message=diagnostic.message, shapes=shapes) - return None - - -def _set_operation_help( - *, - message: str, - authored_text: str, - authored_line: str, - span_column: int, - sql_test: bool, - relation_shapes: dict[str, dict[str, str]], -) -> str | None: - match: re.Match[str] | None = _SET_OPERATION_TYPES.search(message) - if match is None: - return None - position: str = match.group("position") - left: str = match.group("left").upper() - right: str = match.group("right").upper() - column: str = _set_operation_operand( - authored_text=authored_text, - authored_line=authored_line, - column=span_column, - position=position, - ) - if not sql_test: - return ( - "Cast one branch so both branches have the same type, for example " - f"`CAST({column} AS {left})` for set-operation column {position}" - ) - model_type: str | None = _model_type( - authored_text=authored_text, operand=column, relation_shapes=relation_shapes - ) - if model_type is not None: - return ( - f"Cast set-operation column {position} to the tested model's type {model_type} in " - f"the expected or fixture rows, for example `CAST({column} AS {model_type})`" - ) - target: str = right if _is_text(left) and not _is_text(right) else left - return ( - f"Cast set-operation column {position} explicitly so every branch has the same type, " - f"for example `CAST({column} AS {target})` in the expected or fixture rows" - ) - - -def _model_type( - *, authored_text: str, operand: str, relation_shapes: dict[str, dict[str, str]] -) -> str | None: - """The tested relation's type for the branch column, when exactly one type matches.""" - - names: list[str] = [] - alias: re.Match[str] | None = _ALIAS_NAME.search(authored_text.strip()) - if alias is not None: - names.append(alias.group("name")) - names.append(operand.rsplit(".", 1)[-1].strip('"')) - for name in names: - found: set[str] = set() - for shape in relation_shapes.values(): - for output, value in shape.items(): - if output.casefold() == name.casefold() and value: - found.add(value.split("(", 1)[0].upper()) - if len(found) == 1: - return found.pop() - return None - - -def _audit_comparison_help( - *, - message: str, - authored_line: str, - column: int, - authored_body: str, - audit_definition: str | None, - dialect: str | None, -) -> str | None: - match: re.Match[str] | None = _COMPARISON_TYPES.search(message) - if match is None: - return None - types: tuple[str, str] = (match.group("left").upper(), match.group("right").upper()) - temporal: str | None = _temporal_type(types) - if temporal is None or not any(_is_text(value) for value in types): - return None - operand: str = _audit_parameter( - authored_line=authored_line, column=column, authored_body=authored_body - ) - conversion: str = ( - f"TO_{temporal}({operand})" if dialect == _SNOWFLAKE else f"CAST({operand} AS {temporal})" - ) - audit: str = f"audit '{audit_definition}'" if audit_definition else "the audit" - return ( - f"Convert the text value explicitly, for example `{conversion}`, " - f"or attach {audit} to a {temporal} column" - ) - - -def _set_operation_operand( - *, authored_text: str, authored_line: str, column: int, position: str -) -> str: - """The authored branch operand when the span is exactly one, else the VALUES column name.""" - - preceding: str = authored_line[column - 2 : column - 1] if column > 1 else "" - expression: str = _ALIAS_SUFFIX.sub("", authored_text.strip().rstrip(",")) - cast: re.Match[str] | None = _CAST.fullmatch(expression) - if cast is not None: - expression = cast.group("inner").strip() - if ( - not _WORD_CHARACTER.fullmatch(preceding) - and _SINGLE_OPERAND.fullmatch(expression) - and expression.upper() not in _SET_OPERATION_KEYWORDS - ): - return expression - return f"COLUMN{position}" - - -def _audit_parameter(*, authored_line: str, column: int, authored_body: str) -> str: - """The audit parameter nearest the finding on its line, else the body's only parameter.""" - - on_line: list[re.Match[str]] = [ - match - for match in _PARAMETER.finditer(authored_line) - if match.group() != _RELATION_PARAMETER - ] - if on_line: - return min(on_line, key=lambda match: abs(match.start() + 1 - column)).group() - parameters: set[str] = set(_PARAMETER.findall(authored_body)) - {_RELATION_PARAMETER} - return next(iter(parameters)) if len(parameters) == 1 else "" - - -def _unknown_column_help(*, message: str, shapes: dict[str, dict[str, str]]) -> str | None: - missing: tuple[str, str | None] | None = missing_column(message) - if missing is None: - return None - name: str = missing[0] - table: str | None = missing[1] - if table is None: - return None - columns: dict[str, str] = shapes.get(table, {}) - suggestion: str | None = closest_column(name=name, columns=columns) - if suggestion is not None: - return f"did you mean '{suggestion}'?" - listed: str = ", ".join(list(columns)[:_LISTED_COLUMNS]) - more: str = ( - f", and {len(columns) - _LISTED_COLUMNS} more" if len(columns) > _LISTED_COLUMNS else "" - ) - outputs: str = f" (it outputs {listed}{more})" if columns else "" - return f"'{table}' does not output '{name}'{outputs}; update the test to its current columns" - - -def _temporal_type(types: tuple[str, str]) -> str | None: - for value in types: - for kind in _TEMPORAL_TYPES: - if value.startswith(kind): - return kind - return None - - -def _is_text(value: str) -> bool: - return value.startswith(_TEXT_TYPES) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py deleted file mode 100644 index e328348fd4..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/sql_analysis_opt_outs.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Reject `sql_analysis false` on parseable models when the project requires SQL analysis.""" - -from __future__ import annotations - -import re -from collections import Counter -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.analysis.validation import ( - hook_sql_statements, -) -from sqlbuild.compiler.compile.constants import UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, - SqlAnalysisOptOutRequest, -) -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.constants import ( - PROJECT_CONFIG_FILENAME, - REQUIRE_SQL_ANALYSIS_SETTING_KEY, - SETTINGS_SECTION, - SQL_ANALYSIS_CONFIG_KEY, -) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.project_assembly.main._check_native_sql_syntax import ( - check_native_sql_syntax, -) -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.errors.setting_help.main.setting_note import setting_note -from sqlbuild.presentation.main.count_noun import format_count_noun -from sqlbuild.spec.contracts.models import SettingsConfig, SourceLocation - -_LEGACY_KEY: str = "sql_validation" -_OPT_OUT_KEYS: tuple[str, ...] = (SQL_ANALYSIS_CONFIG_KEY, _LEGACY_KEY) -_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") -_OPT_OUT_KEY_PATTERN: re.Pattern[str] = re.compile(r"\b(?:sql_analysis|sql_validation)\b") -_FINDING_KINDS: tuple[tuple[re.Pattern[str], str, str], ...] = ( - (re.compile(r"^B002$"), "unknown column", "unknown columns"), - (re.compile(r"^B003$"), "ambiguous column", "ambiguous columns"), - (re.compile(r"^B00[45]$"), "reference error", "reference errors"), - (re.compile(r"^B101$"), "unknown function", "unknown functions"), - (re.compile(r"^B102$"), "function arity error", "function arity errors"), - (re.compile(r"^B216$"), "column count mismatch", "column count mismatches"), - (re.compile(r"^B21\d$"), "type mismatch", "type mismatches"), - (re.compile(r"^B23[0-2]$"), "grouping or window error", "grouping or window errors"), - (re.compile(r"^B3\d\d$"), "metadata mismatch", "metadata mismatches"), -) - - -def rejected_sql_analysis_opt_out(request: SqlAnalysisOptOutRequest) -> SourceLocation | None: - """Return where a model turns SQL analysis off when the project forbids it for that model.""" - - settings: SettingsConfig = request.settings - if ( - not settings.require_sql_analysis - or not settings.sql_analysis - or request.no_sql_validation - or not any(request.config.values.get(key) is False for key in _OPT_OUT_KEYS) - ): - return None - if not _parses(request): - return None - return _opt_out_location(request) - - -def _parses(request: SqlAnalysisOptOutRequest) -> bool: - """Whether the model query and every hook pass Python's syntax validation.""" - - statements: list[str] = [request.query_sql] - for hook_name in _HOOK_KEYS: - statements.extend(hook_sql_statements(request.config.values.get(hook_name))) - return check_native_sql_syntax( - checks=tuple((statement, request.placeholders) for statement in statements), - dialect=None, - ) - - -def unneeded_opt_out_diagnostic( - *, - resource_type: CompiledResourceType, - kind: str, - name: str, - location: SourceLocation | None, - hidden: tuple[CompilerDiagnostic, ...], - path_default: bool = False, -) -> CompilerDiagnostic: - """One P009 error for `sql_analysis false` on SQL that parses, counting what it hid.""" - - findings: str = _finding_counts(hidden) - opt_out: str = ( - "`sql_analysis = false` from this [path_defaults] entry" - if path_default - else "`sql_analysis false`" - ) - fix: str = ( - f"remove {opt_out} and fix the findings it was hiding: {findings} " - "(run `sqb compile` to see them)" - if findings - else f"remove {opt_out}; it is not hiding any findings" - ) - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE, - message=f"`sql_analysis false` is not needed for {kind} '{name}'", - resource_type=resource_type, - resource_name=name, - location=location, - notes=( - setting_note( - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, - value=True, - ) - + ", which only allows `sql_analysis false` on SQL that SQLBuild cannot parse; " - f"this {kind} parses successfully.", - ), - help=join_helps( - fix, - setting_help( - purpose="to allow `sql_analysis false` on any model, SQL test or audit", - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, - value=False, - ), - ), - ) - - -def _finding_counts(hidden: tuple[CompilerDiagnostic, ...]) -> str: - counts: Counter[tuple[str, str]] = Counter(_finding_kind(item.code) for item in hidden) - return ", ".join( - format_count_noun(count=count, singular=singular, plural=plural) - for (singular, plural), count in sorted(counts.items(), key=lambda item: -item[1]) - ) - - -def _finding_kind(code: str) -> tuple[str, str]: - for pattern, singular, plural in _FINDING_KINDS: - if pattern.match(code): - return singular, plural - return "other finding", "other findings" - - -def _opt_out_location(request: SqlAnalysisOptOutRequest) -> SourceLocation: - model_file: DiscoveredSqlModelFile = request.model_file - if any(key in request.config.model_header_keys for key in _OPT_OUT_KEYS): - return _text_location(path=model_file.relative_path, text=model_file.contents, start=0) - path_key: str | None = request.config.matched_path_default - config_text: str = ( - request.project_config_path.read_text(encoding="utf-8") - if request.project_config_path.is_file() - else "" - ) - start: int = config_text.find(path_key) if path_key else -1 - return _text_location( - path=Path(request.project_config_path.name), text=config_text, start=max(start, 0) - ) - - -def _text_location(*, path: Path, text: str, start: int) -> SourceLocation: - match: re.Match[str] | None = _OPT_OUT_KEY_PATTERN.search(text, start) - offset: int = match.start() if match is not None else 0 - line: int = text.count("\n", 0, offset) + 1 - column: int = offset - (text.rfind("\n", 0, offset) + 1) + 1 - return SourceLocation(path=path, line=line, column=column) diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py index e86357b808..9e06ffcca5 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py @@ -17,17 +17,17 @@ from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledRelationLocation, - CompiledSource, CompilerDiagnostic, PythonSqlReferenceReport, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, + CompiledAuditView, + CompiledModelView, + CompiledProjectView, + CompiledSourceView, DiagnosticPhase, DiagnosticSeverity, ) @@ -71,7 +71,10 @@ class _LiteralSql: def explicit_reference_report( - *, project: CompiledProject, discovered_inputs: DiscoveredProjectInputs, enforce_explicit: bool + *, + project: CompiledProjectView, + discovered_inputs: DiscoveredProjectInputs, + enforce_explicit: bool, ) -> PythonSqlReferenceReport: """Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001.""" @@ -107,7 +110,10 @@ def explicit_reference_report( ], "sources": [_source_row(source) for source in project.sources], "seeds": [ - {"name": seed.name, "relation": _location_row(seed.destination)} + { + "name": seed.name, + "relation": _location_row(seed.destination), + } for seed in project.seeds ], "audits": [_audit_row(audit) for audit in project.audits], @@ -190,13 +196,20 @@ def _owner_row( } -def _resource_rows(project: CompiledProject) -> list[dict[str, object]]: +def _resource_rows(project: CompiledProjectView) -> list[dict[str, object]]: return [ - {"key": _key_row(resource.key), "deps": _key_rows(resource.deps)} - for resource in (*project.models, *project.sources, *project.seeds, *project.functions) + *(_graph_row(key=model.key_row, deps=model.dep_rows) for model in project.models), + *( + _graph_row(key=_key_row(resource.key), deps=_key_rows(resource.deps)) + for resource in (*project.sources, *project.seeds, *project.functions) + ), ] +def _graph_row(*, key: tuple[str, str], deps: list[tuple[str, str]]) -> dict[str, object]: + return {"key": key, "deps": deps} + + def _key_row(key: CompiledObjectKey) -> tuple[str, str]: return (str(key.resource_type), key.name) @@ -205,7 +218,7 @@ def _key_rows(keys: tuple[CompiledObjectKey, ...]) -> list[tuple[str, str]]: return [_key_row(key) for key in keys] -def _model_row(*, model: CompiledModel, old_name_retention: str | None) -> dict[str, object]: +def _model_row(*, model: CompiledModelView, old_name_retention: str | None) -> dict[str, object]: raw: object = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) retained: bool = isinstance(raw, str) and ( resolve_old_name_view_retention( @@ -216,7 +229,7 @@ def _model_row(*, model: CompiledModel, old_name_retention: str | None) -> dict[ return { "name": model.name, "path": str(model.relative_path), - "destination": _location_row(model.destination), + "destination": _destination_row(model.destination_row), "python_hooks": list(model_python_hook_names(model=model)), "sql_hooks": list( chain.from_iterable( @@ -245,7 +258,7 @@ def _sql_hook_rows(*, hook_key: str, entries: object) -> list[dict[str, object]] ] -def _source_row(source: CompiledSource) -> dict[str, object]: +def _source_row(source: CompiledSourceView) -> dict[str, object]: entry: SourceEntry = source.source_entry return { "name": source.name, @@ -256,7 +269,7 @@ def _source_row(source: CompiledSource) -> dict[str, object]: } -def _audit_row(audit: CompiledAudit) -> dict[str, object]: +def _audit_row(audit: CompiledAuditView) -> dict[str, object]: target: tuple[str, str] | None = ( ( str(AttachedAuditTargetKind(audit.attached_target_kind).resource_type), @@ -274,7 +287,12 @@ def _audit_row(audit: CompiledAudit) -> dict[str, object]: def _location_row(location: CompiledRelationLocation) -> dict[str, str | None]: - return {"name": location.name, "schema": location.schema, "database": location.database} + return _destination_row((location.database, location.schema, location.name)) + + +def _destination_row(row: tuple[str | None, str | None, str]) -> dict[str, str | None]: + database, schema, name = row + return {"name": name, "schema": schema, "database": database} def _ref_rows(refs: tuple[SqlResourceRef, ...]) -> list[tuple[str, str]]: @@ -282,7 +300,7 @@ def _ref_rows(refs: tuple[SqlResourceRef, ...]) -> list[tuple[str, str]]: def _owners( - *, project: CompiledProject, discovered_inputs: DiscoveredProjectInputs + *, project: CompiledProjectView, discovered_inputs: DiscoveredProjectInputs ) -> tuple[_PythonSqlOwner, ...]: owners: list[_PythonSqlOwner] = [] for label, nodes in ( @@ -320,7 +338,7 @@ def _owners( def _loader_owners( - *, project: CompiledProject, discovered_inputs: DiscoveredProjectInputs + *, project: CompiledProjectView, discovered_inputs: DiscoveredProjectInputs ) -> tuple[_PythonSqlOwner, ...]: sources_by_loader: dict[str, set[str]] = {} for source in project.sources: diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py index ad05d2f336..856d796075 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py @@ -2,42 +2,15 @@ from __future__ import annotations -import re -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - report_compile_diagnostic, -) from sqlbuild.compiler.compile._helpers.refs.references import ( extract_sql_references, - reference_call_location, - reference_call_syntax_key, - scan_sql_reference_calls, -) -from sqlbuild.compiler.compile.constants import ( - MACRO_GENERATED_REFERENCE_CODE, - REFERENCE_CALL_SYNTAX_CODE, ) -from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, CompileSqlReference, - InvalidSqlReferenceCall, - LoadedMacro, ) -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity -from sqlbuild.compiler.scopes.types import ResourceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceLocation _TYPED_REFERENCE_KINDS: dict[str, SqlResourceRefKind] = { "__ref": SqlResourceRefKind.MODEL, @@ -47,48 +20,10 @@ _REFERENCE_FUNCTION_BY_KIND: dict[SqlResourceRefKind, str] = { kind: function for function, kind in _TYPED_REFERENCE_KINDS.items() } -_EXPLICIT_REFERENCE_KINDS: dict[str, SqlResourceRefKind] = { - SqlReferenceKind.REF.value: SqlResourceRefKind.MODEL, - SqlReferenceKind.SOURCE.value: SqlResourceRefKind.SOURCE, - SqlReferenceKind.SEED.value: SqlResourceRefKind.SEED, -} -_RESOURCE_TYPE_BY_CONSUMER_KIND: dict[ResourceKind, CompiledResourceType] = { - ResourceKind.MODEL: CompiledResourceType.MODEL, - ResourceKind.SOURCE: CompiledResourceType.SOURCE, - ResourceKind.SEED: CompiledResourceType.SEED, -} -_REFERENCE_CALL_MARKERS: tuple[str, ...] = tuple(f"{name}(" for name in _TYPED_REFERENCE_KINDS) -_ALL_REFERENCE_CALL_MARKERS: tuple[str, ...] = tuple( - f"{kind.function_name}(" for kind in SqlReferenceKind -) -_PASSABLE_REFERENCE_KINDS: frozenset[SqlReferenceKind] = frozenset( - {SqlReferenceKind.REF, SqlReferenceKind.SOURCE, SqlReferenceKind.SEED} -) -_RELATION_PLACEHOLDER_PREFIX: str = "__sqlbuild_relation_" -_RELATION_PLACEHOLDER_PATTERN: re.Pattern[str] = re.compile( - rf"{_RELATION_PLACEHOLDER_PREFIX}(\d+)__" -) _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() -def relation_placeholder_text(index: int) -> str: - """Return the in-expansion placeholder text for the typed reference at ``index``.""" - - return f"{_RELATION_PLACEHOLDER_PREFIX}{index}__" - - -def render_relation_placeholders(*, sql: str, relations: dict[SqlResourceRef, int]) -> str: - """Replace placeholders with the reference call written at the macro call site.""" - - if not relations or _RELATION_PLACEHOLDER_PREFIX not in sql: - return sql - ordered: tuple[SqlResourceRef, ...] = tuple(relations) - return _RELATION_PLACEHOLDER_PATTERN.sub( - lambda match: reference_call_text(ordered[int(match.group(1))]), sql - ) - - def reference_call_text(ref: SqlResourceRef) -> str: """Return the SQL reference call that names ``ref``, such as ``__ref("orders")``.""" @@ -107,19 +42,6 @@ def call_site_sql_references( ) -def resource_references( - references: tuple[CompileSqlReference, ...], -) -> tuple[SqlResourceRef, ...]: - """Return the model, source, and seed references among SQL references, once each.""" - - resources: dict[SqlResourceRef, None] = {} - for reference in references: - kind: SqlResourceRefKind | None = _EXPLICIT_REFERENCE_KINDS.get(str(reference.ref_kind)) - if kind is not None: - resources[SqlResourceRef(kind=kind, name=reference.ref_name)] = None - return tuple(resources) - - def merge_call_site_references( *, references: tuple[CompileSqlReference, ...], @@ -132,139 +54,7 @@ def merge_call_site_references( return tuple(dict.fromkeys((*references, *argument_references))) -def reject_macro_generated_references( - *, - loaded_macro: LoadedMacro, - macro_result: str, - file_path: Path, - consumer: ResourceIdentity | DeclarationIdentity | None, -) -> None: - """Report each typed reference a macro emitted instead of receiving it as an argument.""" - - if not any(marker in macro_result for marker in _REFERENCE_CALL_MARKERS): - return - consumer_label: str = ( - f"{consumer.kind.value}:{consumer.name}" if consumer is not None else f"'{file_path}'" - ) - location: str = ( - "the model" - if isinstance(consumer, ResourceIdentity) and consumer.kind is ResourceKind.MODEL - else "the calling SQL" - ) - resource: ResourceIdentity | None = consumer if isinstance(consumer, ResourceIdentity) else None - resource_type: CompiledResourceType | None = ( - _RESOURCE_TYPE_BY_CONSUMER_KIND.get(resource.kind) if resource is not None else None - ) - generated: SqlResourceRef - for generated in _generated_references(macro_result): - report_compile_diagnostic( - key=( - MACRO_GENERATED_REFERENCE_CODE, - consumer_label, - loaded_macro.name, - generated.kind.value, - generated.name, - ), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=MACRO_GENERATED_REFERENCE_CODE, - message=( - f"{consumer_label} depends on {generated.kind.value}:{generated.name} " - f"through macro {loaded_macro.name}()" - ), - resource_type=resource_type, - resource_name=( - resource.name if resource is not None and resource_type is not None else None - ), - path=loaded_macro.relative_path, - help=( - f"write the reference in {location}, or pass it in: " - f"@{loaded_macro.name}({reference_call_text(generated)})\n" - " = help: " + explicit_references_help(allowed="macro-generated references") - ), - ), - ) - - -def report_macro_reference_call_syntax( - *, - loaded_macro: LoadedMacro, - macro_result: str, - file_path: Path, - consumer: ResourceIdentity | DeclarationIdentity | None, -) -> None: - """Report each reference call a macro returned that compile could not replace.""" - - if not any(marker in macro_result for marker in _ALL_REFERENCE_CALL_MARKERS): - return - try: - invalid_calls: tuple[InvalidSqlReferenceCall, ...] = scan_sql_reference_calls( - sql=macro_result, syntax=_GENERIC_SQL_SYNTAX - ).invalid_calls - except CompileInputError: - return - resource: ResourceIdentity | None = consumer if isinstance(consumer, ResourceIdentity) else None - resource_type: CompiledResourceType | None = ( - _RESOURCE_TYPE_BY_CONSUMER_KIND.get(resource.kind) if resource is not None else None - ) - location: str = ( - "the model" - if isinstance(consumer, ResourceIdentity) and consumer.kind is ResourceKind.MODEL - else "the calling SQL" - ) - for invalid_call in invalid_calls: - pass_in_help: str = ( - f"write {invalid_call.corrected_call} in {location} and pass it in: " - f"@{loaded_macro.name}({invalid_call.corrected_call})\n = help: " - if invalid_call.ref_kind in _PASSABLE_REFERENCE_KINDS - else "" - ) - report_compile_diagnostic( - key=reference_call_syntax_key( - file_path=file_path, call=invalid_call.call, location=None - ), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=REFERENCE_CALL_SYNTAX_CODE, - message=f"{invalid_call.message}, returned by macro {loaded_macro.name}()", - resource_type=resource_type, - resource_name=( - resource.name if resource is not None and resource_type is not None else None - ), - path=loaded_macro.relative_path, - location=_macro_source_location(loaded_macro=loaded_macro, call=invalid_call.call), - help=f"{pass_in_help}{invalid_call.help}", - ), - ) - - -def _macro_source_location(*, loaded_macro: LoadedMacro, call: str) -> SourceLocation | None: - start: int = loaded_macro.raw_source.find(call) - if start == -1: - return None - return reference_call_location( - path=loaded_macro.relative_path, text=loaded_macro.raw_source, start=start, call=call - ) - - -def _generated_references(sql: str) -> tuple[SqlResourceRef, ...]: - try: - references: tuple[CompileSqlReference, ...] = scan_sql_reference_calls( - sql=sql, syntax=_GENERIC_SQL_SYNTAX - ).references - except CompileInputError: - return () - generated: dict[SqlResourceRef, None] = {} - for reference in references: - kind: SqlResourceRefKind | None = _EXPLICIT_REFERENCE_KINDS.get(str(reference.ref_kind)) - if kind is not None: - generated[SqlResourceRef(kind=kind, name=reference.ref_name)] = None - return tuple(generated) - - -def typed_reference_value(*, function: str, name: str) -> SqlResourceRef: - """Return the reference a `__ref`, `__source` or `__seed` macro argument names.""" +def typed_reference_kinds() -> dict[str, SqlResourceRefKind]: + """Each typed reference function, such as `__ref`, and the resource kind it names.""" - return SqlResourceRef(kind=_TYPED_REFERENCE_KINDS[function], name=name) + return dict(_TYPED_REFERENCE_KINDS) diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py deleted file mode 100644 index 783a795e89..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/expansion.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Expand one authored string through the native macro bridge, executing misses in Python.""" - -from __future__ import annotations - -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.compile._helpers.render.macros import ( - _expansion_declarations, - _record_macro_declaration_usage, - _record_macro_use, - _report_generated_references, - evaluate_macro_call, - raise_failed_macro_call, -) -from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts -from sqlbuild.compiler.compile.constants import MACRO_TOKEN -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - ExpansionSpan, - MacroExpansionState, -) -from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError -from sqlbuild.compiler.frontier.main._report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite -from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge -from sqlbuild.compiler.macro_bridge.constants import ( - ARGUMENT_REFERENCE_EVENT, - DECLARATION_READ_EVENT, - GENERATED_SQL_EVENT, - MACRO_USE_EVENT, -) -from sqlbuild.compiler.macro_bridge.main._scan_macro_calls import scan_macro_calls -from sqlbuild.compiler.macro_bridge.main._splice_macro_calls import splice_macro_calls -from sqlbuild.compiler.macro_bridge.models import ( - MacroCallClass, - MacroCallScan, - MacroCallSite, -) -from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord -from sqlbuild.compiler.scopes.models import DeclarationIdentity -from sqlbuild.compiler.scopes.types import DeclarationKind -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind - - -def expand_bridged_sql_macros( - *, - sql: str, - consumer_path: Path, - state: MacroExpansionState, - bridge: MacroBridge, -) -> tuple[str, tuple[ExpansionSpan, ...]]: - """Expand one string, replaying memoized calls; unresolved trees evaluate calls in order.""" - - if MACRO_TOKEN not in sql: - return sql, () - declarations: DeclarationResolutionContext | None = _expansion_declarations( - state=state, consumer_path=consumer_path - ) - scan: MacroCallScan = bridge.scan(sql) - sites: tuple[MacroCallSite, ...] = scan.sites - call_classes: list[MacroCallClass | None] = ( - [] - if scan.failure is not None - else [ - bridge.call_class( - site=site, - declarations=declarations, - loaded_macros=state.loaded_macros, - macro_context=state.macro_context, - ) - for site in sites - ] - ) - if scan.failure is not None or None in call_classes: - return _unmemoised_sql_macros( - sql=sql, consumer_path=consumer_path, state=state, declarations=declarations, scan=scan - ) - outputs: list[str] = [ - _bridged_call_output( - sql=sql, - consumer_path=consumer_path, - state=state, - declarations=declarations, - bridge=bridge, - site=site, - call_class=call_class, - ) - for site, call_class in zip(sites, call_classes, strict=True) - if call_class is not None - ] - return bridge.splice(sql=sql, sites=sites, outputs=outputs) - - -def _bridged_call_output( # noqa: PLR0913 - *, - sql: str, - consumer_path: Path, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, - bridge: MacroBridge, - site: MacroCallSite, - call_class: MacroCallClass, -) -> str: - facts: MacroExpansionFacts = state.facts - if state.macro_overrides.keys() & set(site.tree_names or ()): - return _mocked_call_output(sql=sql, consumer_path=consumer_path, state=state, site=site) - call_text: str = sql[site.start : site.end] - prior_relations: tuple[SqlResourceRef, ...] | None = ( - tuple(facts.relations) if site.typed_reference_text else None - ) - class_id: int - recorded: MacroCallRecord | None - class_id, recorded = bridge.lookup( - call_class=call_class, call_text=call_text, prior_relations=prior_relations - ) - if recorded is not None: - recorded_sql, relations, events = recorded - for event in events: - _replay_macro_call_event( - event=event, - consumer_path=consumer_path, - state=state, - declarations=declarations, - ) - for relation in relations: - _ = facts.relations.setdefault(relation, len(facts.relations)) - return facts.render_relation_placeholders(recorded_sql) - known_relations: int = len(facts.relations) - _ = facts.begin_event_log() - try: - macro_result: str = _top_level_output( - sql=sql, - consumer_path=consumer_path, - state=state, - declarations=declarations, - site=site, - ) - finally: - events, replayable = facts.end_event_log() - added_relations: tuple[SqlResourceRef, ...] = tuple(facts.relations)[known_relations:] - if replayable and (prior_relations is not None or not added_relations): - bridge.record( - class_id=class_id, - call_text=call_text, - sql=macro_result, - relations=added_relations, - events=events, - ) - return facts.render_relation_placeholders(macro_result) - - -def expand_unbridged_sql_macros( - *, sql: str, consumer_path: Path, state: MacroExpansionState -) -> tuple[str, tuple[ExpansionSpan, ...]]: - """Expand one string outside compile inputs: native call sites, each evaluated in order.""" - - if MACRO_TOKEN not in sql: - return sql, () - return _unmemoised_sql_macros( - sql=sql, - consumer_path=consumer_path, - state=state, - declarations=_expansion_declarations(state=state, consumer_path=consumer_path), - scan=scan_macro_calls(sql=sql), - ) - - -def _unmemoised_sql_macros( - *, - sql: str, - consumer_path: Path, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, - scan: MacroCallScan, -) -> tuple[str, tuple[ExpansionSpan, ...]]: - """Evaluate every call in order without the memo, raising where Python's scan raises.""" - - outputs: list[str] = [ - _evaluated_call_output( - sql=sql, - consumer_path=consumer_path, - state=state, - declarations=declarations, - site=site, - ) - for site in scan.sites - ] - if scan.failure is not None: - raise_failed_macro_call( - failure=scan.failure, - file_path=consumer_path, - state=state, - declarations=declarations, - ) - return splice_macro_calls(sql=sql, sites=scan.sites, outputs=outputs) - - -def _mocked_call_output( - *, sql: str, consumer_path: Path, state: MacroExpansionState, site: MacroCallSite -) -> str: - """Run a call whose tree a test mocks; its output differs from the shared memo's, so skip it.""" - - report_native_fallback(site=NativeFallbackSite.MACRO_CALL_MOCKED, kind="test_mock") - return _evaluated_call_output( - sql=sql, - consumer_path=consumer_path, - state=state, - declarations=_expansion_declarations(state=state, consumer_path=consumer_path), - site=site, - ) - - -def _evaluated_call_output( - *, - sql: str, - consumer_path: Path, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, - site: MacroCallSite, -) -> str: - """Evaluate one natively scanned call in Python without the memo.""" - - macro_result: str = _top_level_output( - sql=sql, - consumer_path=consumer_path, - state=state, - declarations=declarations, - site=site, - ) - return state.facts.render_relation_placeholders(macro_result) - - -def _top_level_output( - *, - sql: str, - consumer_path: Path, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, - site: MacroCallSite, -) -> str: - """One top-level call's SQL; a non-string result raises as Python's top-level check does.""" - - macro_result: object = evaluate_macro_call( - sql=sql, - site=site, - file_path=consumer_path, - state=state, - declarations=declarations, - stack=(), - top_level=True, - ) - return cast(str, macro_result) - - -def _replay_macro_call_event( - *, - event: MacroCallEvent, - consumer_path: Path, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, -) -> None: - tag, first, second = event - if tag == MACRO_USE_EVENT: - _record_macro_use( - macro_name=first, - identity=( - declarations.macro_records[first].identity - if declarations is not None - else DeclarationIdentity(kind=DeclarationKind.MACRO, name=first) - ), - state=state, - declarations=declarations, - stack=(), - ) - elif tag == DECLARATION_READ_EVENT: - if declarations is None: - raise NativeStageMismatchError("Replayed a declaration read without declarations") - _record_macro_declaration_usage( - name=second, kind=DeclarationKind(first), declarations=declarations, state=state - ) - elif tag == GENERATED_SQL_EVENT: - _report_generated_references( - macro_name=first, - loaded_macro=( - declarations.macros[first] - if declarations is not None - else state.loaded_macros[first] - ), - macro_result=second, - file_path=consumer_path, - state=state, - ) - elif tag == ARGUMENT_REFERENCE_EVENT: - state.facts.record_call_site_refs( - (SqlResourceRef(kind=SqlResourceRefKind(first), name=second),) - ) - else: - raise NativeStageMismatchError(f"Unknown macro call event {tag}") diff --git a/src/sqlbuild/compiler/compile/_helpers/named_declarations/core.py b/src/sqlbuild/compiler/compile/_helpers/named_declarations/core.py deleted file mode 100644 index 62604564bc..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/named_declarations/core.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Visibility and usage facts for audits, reusable schemas, and named hooks used by name.""" - -from __future__ import annotations - -from dataclasses import replace -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.diagnostics.scope import compiled_resource_type -from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_expansion -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, - DeclarationExpansionContext, - DeclarationScopeResolver, -) -from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity -from sqlbuild.compiler.scopes.constants import DECLARATION_ROLE_PARTS -from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path -from sqlbuild.compiler.scopes.main._declaration_visibility import declaration_visibility -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - ResourceIdentity, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeDiagnosticCode - -_KIND_LABELS: dict[DeclarationKind, str] = { - DeclarationKind.AUDIT: "Generic audit", - DeclarationKind.SCHEMA: "Schema", - DeclarationKind.SQL_HOOK: "SQL hook", - DeclarationKind.PYTHON_HOOK: "Python hook", -} - - -def named_declaration_record( - *, resolver: DeclarationScopeResolver | None, kind: DeclarationKind, name: str -) -> DeclarationRecord | None: - """Return the indexed project declaration, or None for built-ins and unindexed inputs.""" - - if resolver is None: - return None - records: tuple[DeclarationRecord, ...] = resolver.lookup.declarations.get( - DeclarationIdentity(kind, name), () - ) - return records[0] if records else None - - -def named_declaration_usages( - *, - resolver: DeclarationScopeResolver | None, - kind: DeclarationKind, - name: str, - consumer: ResourceIdentity | DeclarationIdentity, - consumer_path: Path | DeclarationRecord, -) -> tuple[UsageRecord, ...]: - """Require a declaration to be visible from its consumer and return the usage fact.""" - - record: DeclarationRecord | None = named_declaration_record( - resolver=resolver, kind=kind, name=name - ) - if record is None: - return () - if ( - isinstance(consumer_path, Path) - and consumer_path.is_absolute() - and resolver is not None - and resolver.project_dir is not None - and consumer_path.is_relative_to(resolver.project_dir) - ): - consumer_path = consumer_path.relative_to(resolver.project_dir) - lexical_consumer: str | DeclarationRecord = ( - consumer_path if isinstance(consumer_path, DeclarationRecord) else consumer_path.as_posix() - ) - if declaration_visibility(declaration=record, consumer=lexical_consumer) is None: - consumer_label: str = ( - consumer_path.path - if isinstance(consumer_path, DeclarationRecord) - else consumer_path.as_posix() - ) - consumer_folder: str = ( - declaration_lexical_path(record=consumer_path).rsplit("/", maxsplit=1)[0] - if isinstance(consumer_path, DeclarationRecord) - else consumer_path.parent.as_posix() - ) - owner: str = record.owning_path or record.ownership_root.path - role: str = "/".join(DECLARATION_ROLE_PARTS[kind]) - resource: ResourceIdentity | None = ( - consumer if isinstance(consumer, ResourceIdentity) else None - ) - report_compile_diagnostic( - key=( - ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value, - consumer_label, - kind.value, - name, - ), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value, - message=( - f"{_KIND_LABELS[kind]} '{name}' is not visible from '{consumer_label}'. It " - f"is defined at {record.path} with {record.scope.value} scope owned by " - f"'{owner}'" - ), - resource_type=( - compiled_resource_type(resource.kind) if resource is not None else None - ), - resource_name=resource.name if resource is not None else None, - path=Path(consumer_label), - help=( - f"move '{record.path}' to a {role}/ role owned by '{consumer_folder}' or one " - f"of its parent folders, or to the project-wide {role}/ when consumers span " - "resource trees" - ), - ), - ) - return (UsageRecord(consumer=consumer, declaration=record.identity),) - - -def declaration_file_expansion( - *, context: DeclarationExpansionContext, file_path: Path, consumer: DeclarationIdentity -) -> DeclarationExpansionContext: - """Resolve macros, enums, and constants from a declaration file's own owning folder.""" - - resolved: DeclarationExpansionContext = resolve_declaration_expansion( - context=context, file_path=file_path - ) - return replace(resolved, declarations=replace(resolved.declarations, consumer=consumer)) diff --git a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py b/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py index 4e175ca41d..ad4f50f1d8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py +++ b/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py @@ -2,13 +2,11 @@ from __future__ import annotations +from itertools import chain from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.named_declarations.core import ( - named_declaration_record, - named_declaration_usages, -) from sqlbuild.compiler.compile.models import CompilerDiagnostic, DeclarationScopeResolver from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity from sqlbuild.compiler.discovery.models import ( @@ -16,14 +14,19 @@ EnumDeclaration, ModelSchemaDeclaration, ) -from sqlbuild.compiler.scopes.main._declaration_visibility import declaration_visibility +from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, + ResourceIdentity, UsageRecord, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeDiagnosticCode +_SCHEMA_USAGE_KINDS: frozenset[DeclarationKind] = frozenset( + {DeclarationKind.SCHEMA, DeclarationKind.ENUM} +) + def model_schema_scope_usages( *, @@ -34,31 +37,62 @@ def model_schema_scope_usages( if resolver is None: return () - usages: list[UsageRecord] = [] - for schema_file in schema_files: - for declaration in schema_file.declarations: - record: DeclarationRecord | None = named_declaration_record( - resolver=resolver, kind=DeclarationKind.SCHEMA, name=declaration.name - ) - if record is None: - continue - if declaration.extends is not None: - usages.extend( - named_declaration_usages( - resolver=resolver, - kind=DeclarationKind.SCHEMA, - name=declaration.extends, - consumer=record.identity, - consumer_path=record, - ) - ) - usages.extend( - UsageRecord(consumer=record.identity, declaration=enum_record.identity) - for enum_record in _enum_type_records( - declaration=declaration, record=record, resolver=resolver - ) - ) - return tuple(dict.fromkeys(usages)) + records: list[DeclarationRecord] = [ + found[0] + for identity, found in resolver.lookup.declarations.items() + if found and identity.owner is None and identity.kind in _SCHEMA_USAGE_KINDS + ] + usages: list[tuple[int, int]] + diagnostics: list[tuple[list[str], str, str, str]] + error: str | None + usages, diagnostics, error = _native.scope_schema_usages( + list(map(_schema_row, chain.from_iterable(item.declarations for item in schema_files))), + list(map(_record_row, records)), + ) + if error is not None: + raise InvalidScopePathError(error) + for key, message, path, help_text in diagnostics: + report_compile_diagnostic( + key=tuple(key), + diagnostic=CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value, + message=message, + path=Path(path), + help=help_text, + ), + ) + return tuple( + UsageRecord(consumer=records[consumer].identity, declaration=records[used].identity) + for consumer, used in usages + ) + + +def _schema_row( + declaration: ModelSchemaDeclaration, +) -> tuple[str, str | None, list[tuple[str, str | None]]]: + return ( + declaration.name, + declaration.extends, + [(column.name, column.type) for column in declaration.columns], + ) + + +def _record_row( + record: DeclarationRecord, +) -> tuple[tuple[str, str], tuple[str, str, tuple[str, str] | None, str | None, str]]: + owner: ResourceIdentity | None = record.identity.owner + return ( + (record.identity.kind.value, record.identity.name), + ( + record.path, + record.scope.value, + None if owner is None else (owner.kind.value, owner.name), + record.owning_path, + record.ownership_root.path, + ), + ) def model_schema_enum_declarations( @@ -78,48 +112,3 @@ def model_schema_enum_declarations( if isinstance(value, EnumDeclaration): enums[column.type] = value return enums - - -def _enum_type_records( - *, - declaration: ModelSchemaDeclaration, - record: DeclarationRecord, - resolver: DeclarationScopeResolver, -) -> tuple[DeclarationRecord, ...]: - records: list[DeclarationRecord] = [] - for column in declaration.columns: - if column.type is None: - continue - enum_record: DeclarationRecord | None = named_declaration_record( - resolver=resolver, kind=DeclarationKind.ENUM, name=column.type - ) - if enum_record is None: - continue - if declaration_visibility(declaration=enum_record, consumer=record) is None: - owner: str = enum_record.owning_path or enum_record.ownership_root.path - report_compile_diagnostic( - key=( - ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value, - record.path, - declaration.name, - column.name, - ), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value, - message=( - f"Schema '{declaration.name}' in {record.path} column '{column.name}' " - f"uses enum '{column.type}', which is not visible from the schema's " - f"location. The enum is defined at {enum_record.path} with " - f"{enum_record.scope.value} scope owned by '{owner}'" - ), - path=Path(record.path), - help=( - "move the enum to a role visible from the schema's owning folder, or " - "move the schema next to the enum" - ), - ), - ) - records.append(enum_record) - return tuple(records) diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index cef32f45df..78eb928af8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -19,13 +19,10 @@ NativeModelAnalysisRequest, NativePivotTables, ) +from sqlbuild.compiler.analysis_session.types import ProofRow from sqlbuild.compiler.compile._helpers.analysis.pivot_requests import standalone_pivot_models from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import model_syntax_checks -from sqlbuild.compiler.compile.models import ( - CompileProjectInputs, - DynamicColumnContractProof, - ModelSqlAnalysis, -) +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.project_assembly.main._assemble_native_project_resources import ( assemble_native_project_resources, @@ -40,10 +37,10 @@ def project_facts_by_engine( *, - inputs: CompileProjectInputs, + inputs: CompileInputsView, dialect: str | None, analysis_model_names: frozenset[str] | None, - analyses: dict[str, ModelSqlAnalysis], + analyses: NativeModelAnalyses | None, session: Any | None, column_types_by_table: dict[str, dict[str, str]], authoritative_column_types_by_table: dict[str, dict[str, str]], @@ -58,17 +55,13 @@ def project_facts_by_engine( syntax_checks=model_syntax_checks( model_inputs=inputs.model_inputs, analysis_model_names=analysis_model_names, - analysis_succeeded=frozenset( - name - for name, analysis in analyses.items() - if analysis.polyglot_analysis.analysis_succeeded - ), + analysis_succeeded=frozenset() if analyses is None else analyses.succeeded, ), ) pivots: dict[int, tuple[str, tuple[SchemaDynamicColumnFamily, ...]]] = standalone_pivot_models( - model_inputs=inputs.model_inputs, analyses=analyses + model_inputs=inputs.model_inputs, analysed={} if analyses is None else analyses.positions ) - proofs: dict[int, DynamicColumnContractProof | None] = dict( + proofs: dict[int, ProofRow | None] = dict( zip( pivots, prove_native_dynamic_contracts( @@ -106,10 +99,7 @@ def expression_source_shapes_by_engine( return infer_native_expression_source_shapes(expressions=expressions, profile=profile) -def analyze_model_sql( - request: NativeModelAnalysisRequest, -) -> tuple[dict[str, ModelSqlAnalysis], Any | None]: - """Analyze models natively; also return the finished session.""" +def analyze_model_sql(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: + """Analyze models natively; their outcomes stay in the finished session.""" - native: NativeModelAnalyses = analyze_native_model_sql(request=request) - return native.analyses, native.session + return analyze_native_model_sql(request=request) diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py index 0f257e5a5f..0435068ead 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py @@ -13,9 +13,9 @@ CompiledModelSqlTestPayload, CompiledSqlTest, CompileModelSqlTestInputPayload, - CompileProjectInputs, CompileSqlTestInput, ) +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.frontier.main._report_native_fallback import report_native_fallback from sqlbuild.compiler.frontier.types import NativeFallbackSite from sqlbuild.compiler.sql_test_glue.constants import ( @@ -31,7 +31,7 @@ def assemble_sql_tests_by_engine( *, - inputs: CompileProjectInputs, + inputs: CompileInputsView, macro_mock_queries: Callable[[CompileSqlTestInput], dict[str, str]], ) -> tuple[CompiledSqlTest, ...]: """Assemble every SQL test natively, raising errors in Python's assembly order.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/cache.py b/src/sqlbuild/compiler/compile/_helpers/refs/cache.py deleted file mode 100644 index a47ec6bca1..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/refs/cache.py +++ /dev/null @@ -1,274 +0,0 @@ -"""Versioned project-local cache for SQL reference extraction.""" - -from __future__ import annotations - -import hashlib -import hmac -import json -import sqlite3 -from collections.abc import Iterator -from contextlib import contextmanager -from pathlib import Path -from types import TracebackType -from typing import Any, cast - -from sqlbuild.compiler.compile._helpers.refs.references import ( - report_invalid_reference_calls, - scan_sql_reference_calls, -) -from sqlbuild.compiler.compile.exceptions import AnalysisCacheEntryError -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, - SqlReferenceOrigin, - SqlReferenceScan, -) -from sqlbuild.compiler.compile.types import SqlReferenceExtractor -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -_REFERENCE_CACHE_VERSION: int = 3 -_CACHE_DATABASE_NAME: str = "sql-references.sqlite3" -_CACHE_ENTRY_SEPARATOR: str = "\n" -_MAX_CACHE_ENTRY_BYTES: int = 1_000_000 -_SQLITE_TIMEOUT_SECONDS: float = 0.1 -_CREATE_CACHE_TABLE_SQL: str = """ -CREATE TABLE IF NOT EXISTS sql_reference ( - cache_key TEXT PRIMARY KEY, - payload TEXT NOT NULL -) -""" - - -class _SqlReferenceCache: - """Reuse exact SQL reference facts while preserving safe scanner fallback.""" - - def __init__(self, *, root: Path | None, syntax: SqlLexicalSyntax) -> None: - self._root: Path | None = root - self._syntax: SqlLexicalSyntax = syntax - self._syntax_key: str = syntax.cache_key - self._database_path: Path | None = None - self._connection: sqlite3.Connection | None = None - self._pending_contents_by_key: dict[str, str] = {} - - def __enter__(self) -> _SqlReferenceCache: - if self._root is None: - return self - self._database_path = ( - self._root / f"references-v{_REFERENCE_CACHE_VERSION}" / _CACHE_DATABASE_NAME - ) - if not self._database_path.is_file(): - return self - try: - self._connection = sqlite3.connect( - self._database_path, - timeout=_SQLITE_TIMEOUT_SECONDS, - ) - except (OSError, sqlite3.DatabaseError): - self._disable() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - del exc_value, traceback - connection: sqlite3.Connection | None = self._connection - self._connection = None - try: - if exc_type is None: - connection = self._write_pending(connection=connection) - elif connection is not None: - connection.rollback() - except (OSError, sqlite3.DatabaseError): - pass - finally: - self._pending_contents_by_key.clear() - if connection is not None: - try: - connection.close() - except sqlite3.DatabaseError: - pass - - def references(self, *, sql: str, origin: SqlReferenceOrigin | None) -> SqlReferenceScan: - """Return cached references, or scan, report rejected calls, and cache clean SQL.""" - - cache_key: str = _reference_cache_key(sql=sql, syntax_key=self._syntax_key) - pending_contents: str | None = self._pending_contents_by_key.get(cache_key) - if pending_contents is not None: - pending_references: tuple[CompileSqlReference, ...] | None = _references_from_contents( - contents=pending_contents, - expected_cache_key=cache_key, - ) - if pending_references is not None: - return SqlReferenceScan(references=pending_references) - connection: sqlite3.Connection | None = self._connection - if connection is not None: - try: - row: tuple[str] | None = connection.execute( - "SELECT payload FROM sql_reference WHERE cache_key = ?", - (cache_key,), - ).fetchone() - if row is not None: - cached_references: tuple[CompileSqlReference, ...] | None = ( - _references_from_contents(contents=row[0], expected_cache_key=cache_key) - ) - if cached_references is not None: - return SqlReferenceScan(references=cached_references) - except sqlite3.DatabaseError: - self._disable() - connection = None - - scan: SqlReferenceScan = scan_sql_reference_calls( - sql=sql, syntax=self._syntax, origin=origin - ) - report_invalid_reference_calls( - invalid_calls=scan.invalid_calls, origin=origin, syntax=self._syntax - ) - if self._database_path is not None and not scan.invalid_calls: - try: - contents: str = _reference_contents( - cache_key=cache_key, - references=scan.references, - ) - if len(contents.encode()) <= _MAX_CACHE_ENTRY_BYTES: - self._pending_contents_by_key[cache_key] = contents - except (TypeError, ValueError): - pass - return scan - - def _write_pending(self, *, connection: sqlite3.Connection | None) -> sqlite3.Connection | None: - if not self._pending_contents_by_key or self._database_path is None: - return connection - if connection is None: - self._database_path.parent.mkdir(parents=True, exist_ok=True) - connection = sqlite3.connect( - self._database_path, - timeout=_SQLITE_TIMEOUT_SECONDS, - ) - _ = connection.execute(_CREATE_CACHE_TABLE_SQL) - _ = connection.executemany( - "INSERT OR REPLACE INTO sql_reference (cache_key, payload) VALUES (?, ?)", - self._pending_contents_by_key.items(), - ) - connection.commit() - return connection - - def _disable(self) -> None: - connection: sqlite3.Connection | None = self._connection - self._connection = None - if connection is None: - return - try: - connection.close() - except sqlite3.DatabaseError: - pass - - -def _reference_cache_key(*, sql: str, syntax_key: str) -> str: - digest: Any = hashlib.sha256(syntax_key.encode()) - digest.update(b"\0") - digest.update(sql.encode()) - return digest.hexdigest() - - -def _reference_contents(*, cache_key: str, references: tuple[CompileSqlReference, ...]) -> str: - serialized_payload: str = json.dumps( - { - "v": _REFERENCE_CACHE_VERSION, - "k": cache_key, - "r": [ - [ - str(reference.ref_kind), - reference.ref_name, - reference.ref_package, - reference.call_argument_count, - ] - for reference in references - ], - }, - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ) - return _CACHE_ENTRY_SEPARATOR.join( - ( - _cache_entry_digest(cache_key=cache_key, serialized_payload=serialized_payload), - serialized_payload, - ) - ) - - -def _references_from_contents( - *, contents: object, expected_cache_key: str -) -> tuple[CompileSqlReference, ...] | None: - if not isinstance(contents, str): - return None - try: - if len(contents.encode()) > _MAX_CACHE_ENTRY_BYTES: - return None - stored_digest, separator, serialized_payload = contents.partition(_CACHE_ENTRY_SEPARATOR) - if not separator or not hmac.compare_digest( - stored_digest, - _cache_entry_digest( - cache_key=expected_cache_key, - serialized_payload=serialized_payload, - ), - ): - return None - payload: object = json.loads(serialized_payload) - if not isinstance(payload, dict): - raise AnalysisCacheEntryError("reference cache entry must be an object") - values: dict[str, Any] = cast(dict[str, Any], payload) - if values["v"] != _REFERENCE_CACHE_VERSION: - raise AnalysisCacheEntryError("reference cache version mismatch") - if values["k"] != expected_cache_key: - raise AnalysisCacheEntryError("reference cache key mismatch") - references_payload: object = values["r"] - if not isinstance(references_payload, list) or not all( - isinstance(reference, list) for reference in references_payload - ): - raise AnalysisCacheEntryError("reference cache facts must be arrays") - return tuple( - _reference_from_payload(cast(list[object], reference)) - for reference in references_payload - ) - except (ValueError, TypeError, KeyError, RecursionError, json.JSONDecodeError): - return None - - -def _reference_from_payload(payload: list[object]) -> CompileSqlReference: - reference_value_count: int = 4 - if len(payload) != reference_value_count: - raise AnalysisCacheEntryError("reference cache fact must contain four values") - kind, name, package, call_argument_count = payload - if not isinstance(kind, str) or not isinstance(name, str): - raise AnalysisCacheEntryError("reference cache kind and name must be strings") - if package is not None and not isinstance(package, str): - raise AnalysisCacheEntryError("reference cache package must be a string or null") - if call_argument_count is not None and type(call_argument_count) is not int: - raise AnalysisCacheEntryError("reference cache argument count must be an integer or null") - return CompileSqlReference( - ref_kind=SqlReferenceKind(kind), - ref_name=name, - ref_package=package, - call_argument_count=call_argument_count, - ) - - -def _cache_entry_digest(*, cache_key: str, serialized_payload: str) -> str: - digest: Any = hashlib.sha256(cache_key.encode()) - digest.update(b"\0") - digest.update(serialized_payload.encode()) - return digest.hexdigest() - - -@contextmanager -def cached_sql_reference_extractor( - *, root: Path | None, syntax: SqlLexicalSyntax -) -> Iterator[SqlReferenceExtractor]: - """Yield an exact cached reference extractor for one compile invocation and dialect.""" - - with _SqlReferenceCache(root=root, syntax=syntax) as cache: - yield cache.references diff --git a/src/sqlbuild/compiler/compile/_helpers/render/arguments.py b/src/sqlbuild/compiler/compile/_helpers/render/arguments.py deleted file mode 100644 index 8d9cfdb802..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/arguments.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Rendering for reusable SQL resource arguments.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.compile.constants import ( - SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN, - SQL_ARGUMENT_RAW_PARAMETER_PATTERN, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError - - -def render_parameterized_sql( - *, - sql: str, - arguments: dict[str, object], - owner_label: str, - definition_label: str, - reject_unused: bool = False, -) -> str: - """Render raw and quoted arguments into reusable SQL text.""" - - matches: list[tuple[re.Match[str], bool]] = [] - match: re.Match[str] - for match in SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN.finditer(sql): - matches.append((match, True)) - for match in SQL_ARGUMENT_RAW_PARAMETER_PATTERN.finditer(sql): - matches.append((match, False)) - matches.sort(key=lambda item: item[0].start()) - referenced_argument_names: set[str] = { - argument_match.group("name") for argument_match, _quoted in matches - } - referenced_arguments: frozenset[str] = frozenset(referenced_argument_names) - if reject_unused: - unused_arguments: tuple[str, ...] = tuple(sorted(arguments.keys() - referenced_arguments)) - if unused_arguments: - raise CompileInputError( - f"{owner_label} passes unknown argument(s) to {definition_label}: " - f"{', '.join(unused_arguments)}" - ) - - rendered_parts: list[str] = [] - previous_end: int = 0 - quoted: bool - for match, quoted in matches: - rendered_parts.append(sql[previous_end : match.start()]) - rendered_parts.append( - _render_argument( - argument_name=match.group("name"), - arguments=arguments, - owner_label=owner_label, - definition_label=definition_label, - quoted=quoted, - ) - ) - previous_end = match.end() - rendered_parts.append(sql[previous_end:]) - return "".join(rendered_parts) - - -def _render_argument( - *, - argument_name: str, - arguments: dict[str, object], - owner_label: str, - definition_label: str, - quoted: bool, -) -> str: - if argument_name not in arguments: - raise CompileInputError( - f"{owner_label} is missing argument '{argument_name}' for {definition_label}" - ) - return render_sql_argument_value( - argument_value=arguments[argument_name], - owner_label=owner_label, - definition_label=definition_label, - argument_name=argument_name, - quoted=quoted, - ) - - -def render_sql_argument_value( - *, - argument_value: object, - owner_label: str, - definition_label: str, - argument_name: str, - quoted: bool, -) -> str: - """Render one argument value using raw or SQL-literal semantics.""" - - if isinstance(argument_value, list | tuple): - return ", ".join( - render_sql_argument_value( - argument_value=item, - owner_label=owner_label, - definition_label=definition_label, - argument_name=argument_name, - quoted=quoted, - ) - for item in argument_value - ) - if isinstance(argument_value, bool): - return "TRUE" if argument_value else "FALSE" - if argument_value is None: - return "NULL" - if isinstance(argument_value, int | float): - return str(argument_value) - if isinstance(argument_value, str): - if quoted: - return f"'{argument_value.replace(chr(39), chr(39) * 2)}'" - return argument_value - raise CompileInputError( - f"{owner_label} {definition_label} argument '{argument_name}' uses an unsupported value" - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py index e81a0725dc..5160e57304 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py @@ -7,11 +7,7 @@ from uuid import uuid4 from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data -from sqlbuild.compiler.compile.constants import ( - PRESERVE_TARGET_VALUE, -) from sqlbuild.compiler.compile.types import CompileContextKey -from sqlbuild.spec.contracts.models import TargetConfig def resolve_early_model_templates( @@ -40,131 +36,6 @@ def resolve_early_model_templates( ) -def resolve_model_context_templates( - *, - values: dict[str, object], - model_name: str, - effective_target_name: str | None, - run_id: str, -) -> dict[str, object]: - """Resolve model-bound `CTX` values once logical model identity is known.""" - - return cast( - dict[str, object], - expand_config_templates( - value=values, - variables={}, - context_values=build_model_context_values( - values=values, - model_name=model_name, - effective_target_name=effective_target_name, - run_id=run_id, - include_target_values=False, - ), - context_label="model config", - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=True, - ), - ) - - -def resolve_chained_model_context_templates( - *, - values: dict[str, object], - model_name: str, - effective_target_name: str | None, - run_id: str, -) -> dict[str, object]: - """Resolve twice so ${CTX:model.*} values may chain exactly one level without looping.""" - - first_pass_values: dict[str, object] = resolve_model_context_templates( - values=values, - model_name=model_name, - effective_target_name=effective_target_name, - run_id=run_id, - ) - return resolve_model_context_templates( - values=first_pass_values, - model_name=model_name, - effective_target_name=effective_target_name, - run_id=run_id, - ) - - -def resolve_target_context_templates( - *, - values: dict[str, object], - model_name: str, - effective_target_name: str | None, - run_id: str, -) -> dict[str, object]: - """Resolve late `target.*` values after environment overrides finalize naming.""" - - return cast( - dict[str, object], - expand_config_templates( - value=values, - variables={}, - context_values=build_model_context_values( - values=values, - model_name=model_name, - effective_target_name=effective_target_name, - run_id=run_id, - include_target_values=True, - ), - context_label="model config", - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=False, - ), - ) - - -def build_model_context_values( - *, - values: dict[str, object], - model_name: str, - effective_target_name: str | None, - run_id: str, - include_target_values: bool, -) -> dict[str, str | None]: - """Build the currently available model-scoped CTX values.""" - - raw_database: object | None = values.get("database") - raw_schema: object | None = values.get("schema") - raw_alias: object | None = values.get("alias") - logical_database: str | None = None if not isinstance(raw_database, str) else raw_database - logical_schema: str | None = None if not isinstance(raw_schema, str) else raw_schema - logical_alias: str = model_name if not isinstance(raw_alias, str) else raw_alias - context_values: dict[str, str | None] = { - **build_run_context_values( - effective_target_name=effective_target_name, - run_id=run_id, - ), - CompileContextKey.MODEL_NAME: model_name, - CompileContextKey.MODEL_DATABASE: logical_database, - CompileContextKey.MODEL_SCHEMA: logical_schema, - CompileContextKey.MODEL_ALIAS: logical_alias, - } - if not include_target_values: - return context_values - - destination_database: str | None = logical_database - destination_schema: str | None = logical_schema - destination_table: str = logical_alias - destination_qualified: str | None = None - if destination_database is not None and destination_schema is not None: - destination_qualified = f"{destination_database}.{destination_schema}.{destination_table}" - elif destination_schema is not None: - destination_qualified = f"{destination_schema}.{destination_table}" - context_values[CompileContextKey.DESTINATION_DATABASE] = destination_database - context_values[CompileContextKey.DESTINATION_SCHEMA] = destination_schema - context_values[CompileContextKey.DESTINATION_TABLE] = destination_table - context_values[CompileContextKey.DESTINATION_QUALIFIED] = destination_qualified - return context_values - - def build_run_context_values( *, effective_target_name: str | None, run_id: str ) -> dict[str, str | None]: @@ -176,42 +47,6 @@ def build_run_context_values( } -def apply_environment_database_schema_overrides( - *, - values: dict[str, object], - effective_vars: dict[str, object], - target_config: TargetConfig | None, - model_context_values: dict[str, str | None], -) -> dict[str, object]: - """Return values with environment database/schema overrides applied.""" - - if target_config is None: - return dict(values) - - overridden: dict[str, object] = dict(values) - if target_config.database is not None and target_config.database != PRESERVE_TARGET_VALUE: - overridden["database"] = expand_config_templates( - value=target_config.database, - variables=effective_vars, - context_values=model_context_values, - context_label="environment database", - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=False, - ) - if target_config.schema is not None and target_config.schema != PRESERVE_TARGET_VALUE: - overridden["schema"] = expand_config_templates( - value=target_config.schema, - variables=effective_vars, - context_values=model_context_values, - context_label="environment schema", - allow_context=True, - preserve_context_tokens=False, - preserve_unknown_context=False, - ) - return overridden - - def resolve_run_id(*, selected_run_id: str | None) -> str: """Resolve a stable compile invocation id.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py b/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py index dcfc1b6832..018469b22b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/cursor_intrinsics.py @@ -38,16 +38,6 @@ def get_validated_model_cursor_intrinsics( return str(canonical_sql) -def reject_cursor_intrinsics(*, sql: str, context: str) -> None: - """Reject cursor intrinsics in SQL that does not own an execution interval.""" - - error: str | None = _native.cursor_intrinsics_rejection( - sql, _RESERVED_MARKERS, context, _PYTHON - ) - if error is not None: - raise CompileInputError(error) - - def render_cursor_intrinsics(*, sql: str, start_sql: str, end_sql: str) -> str: """Render recognized intrinsics to complete adapter-specific bound expressions.""" @@ -82,3 +72,9 @@ def _replaced(*, sql: str, context: str, start_sql: str, end_sql: str) -> tuple[ if error is not None: raise CompileInputError(error) return replaced, found + + +def cursor_intrinsic_semantics() -> tuple[list[str], tuple[tuple[int, int], str]]: + """The reserved markers and Python release native cursor-intrinsic validation reads.""" + + return list(_RESERVED_MARKERS), _PYTHON diff --git a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py index 25c57555fe..346c751640 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/declarations.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/declarations.py @@ -4,13 +4,13 @@ from collections.abc import Mapping from dataclasses import replace +from functools import partial from pathlib import Path from types import MappingProxyType -from typing import cast +import sqlbuild._native as _native from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( - DeclarationExpansionContext, DeclarationExpansionResult, DeclarationResolutionContext, DeclarationRuntimeProjection, @@ -27,59 +27,35 @@ DiscoveredProjectInputs, DiscoveredSqlModelFile, EnumDeclaration, - EnumMember, ModelSchemaDeclaration, ) +from sqlbuild.compiler.frontier.main._native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.main._report_native_fallback import report_native_fallback -from sqlbuild.compiler.frontier.types import NativeFallbackSite -from sqlbuild.compiler.model_loop.constants import ( - ENUM_REFERENCE_KIND_CODE, - INVALID_CONSTANT_REFERENCE_STOP_CODE, - INVALID_ENUM_REFERENCE_STOP_CODE, - UNCLOSED_BLOCK_COMMENT_STOP_CODE, -) +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage from sqlbuild.compiler.model_loop.main._build_native_declaration_contexts import ( build_native_declaration_contexts, ) -from sqlbuild.compiler.model_loop.main._scan_native_declaration_references import ( - scan_native_declaration_references, -) from sqlbuild.compiler.model_loop.types import NativeDeclarationScan -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.compiler.scopes.constants import CURRENT_PATH_COMPONENT, QUALIFIED_IDENTITY_SEPARATOR -from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path -from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( - resolve_scope_declaration_visibility, -) from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, - DeclarationRecord, - OwnershipRoot, ResourceIdentity, - ResourceRecord, ScopeIndex, ScopeLookup, - ScopeTargetQuery, UsageRecord, - VisibilityRecord, ) from sqlbuild.compiler.scopes.types import ( DeclarationKind, ResourceKind, ScopeKind, - UsageKind, ) -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaColumn, SchemaModelEntry +from sqlbuild.spec.contracts.models import SchemaColumn from sqlbuild.sql_values.exceptions import SqlValueRenderingError, SqlValueValidationError from sqlbuild.sql_values.main.validate_rendered_size import validate_rendered_sql_value_size from sqlbuild.sql_values.types import CollectionRendering, SqlValueKind -_CONTEXT: str = "Enum and constant expansion" _ACCEPTED_VALUES_AUDIT: str = "accepted_values" -_ENUM_REFERENCE_KIND: str = "enum" -_CONSTANT_REFERENCE_KIND: str = "const" -_PATH_CACHE_PREFIX: str = "" def build_public_declaration_indexes( @@ -151,7 +127,7 @@ def build_declaration_scope_resolver( if loaded_macros is not None: for macro in loaded_macros.values(): declarations[DeclarationIdentity(DeclarationKind.MACRO, macro.name)] = macro - if lookup is None: + if lookup is None and native_stage_enabled(NativeStage.DECLARATION_SCOPES): report_native_fallback(site=NativeFallbackSite.SCOPE_REBIND_LOOKUP) scope_lookup: ScopeLookup = build_scope_lookup(index=scope_index) if lookup is None else lookup return DeclarationScopeResolver( @@ -178,170 +154,14 @@ def resolve_declaration_context( ) -> DeclarationResolutionContext: """Project canonical visibility for one authored path onto runtime declaration values.""" - target_path: Path = file_path - if resolver.project_dir is not None and file_path.is_absolute(): - try: - target_path = file_path.relative_to(resolver.project_dir) - except ValueError: - target_path = file_path - resources: tuple[ResourceRecord, ...] = ( - resolver.lookup.resources.get(resource, ()) - if resource is not None - else resolver.lookup.resources_by_path.get(target_path.as_posix(), ()) - ) - cache_key: tuple[str, str] | None = None - if ( - len(resources) == 1 - and resolver.resource_specific is not None - and resources[0].identity not in resolver.resource_specific - ): - cache_key = (Path(resources[0].path).parent.as_posix(), resources[0].ownership_root.path) - cached: DeclarationResolutionContext | None = resolver.contexts_by_directory.get(cache_key) - if cached is not None: - return _rebind_declaration_context(context=cached, consumer=resources[0].identity) - elif not resources and resource is None: - cache_key = (_PATH_CACHE_PREFIX, target_path.as_posix()) - cached_path_context: DeclarationResolutionContext | None = ( - resolver.contexts_by_directory.get(cache_key) - ) - if cached_path_context is not None: - return cached_path_context - matches: tuple[ResourceRecord, ...] = resources - consumer: ResourceIdentity | None = resource or (resources[0].identity if resources else None) - if not resources or _scope_query_parses_identity(resource=resource, target_path=target_path): - matches, consumer = _queried_matches( - resolver=resolver, target_path=target_path, resource=resource - ) - context: DeclarationResolutionContext = resolver.native_contexts.context( - list(matches), consumer - ) - if cache_key is not None: - resolver.cache_context(key=cache_key, context=context) + context: DeclarationResolutionContext + projected: bool + context, projected = resolver.native_contexts.resolve(resolver, file_path, resource) + if projected: + report_native_answer(stage=NativeStage.MODEL_LOOP, kind="declaration_contexts") return context -def _scope_query_parses_identity(*, resource: ResourceIdentity | None, target_path: Path) -> bool: - """Whether the scope query reads the path target as a qualified identity.""" - - return resource is None and QUALIFIED_IDENTITY_SEPARATOR in str(target_path) - - -def _queried_matches( - *, - resolver: DeclarationScopeResolver, - target_path: Path, - resource: ResourceIdentity | None, -) -> tuple[tuple[ResourceRecord, ...], ResourceIdentity | None]: - """The resources a scope query matches, or an unknown path's lexical folder record.""" - - query: ScopeTargetQuery = resolve_scope_declaration_visibility( - lookup=resolver.lookup, target=resource or target_path - ).target - if not query.unknown: - return query.matches, resource or (query.matches[0].identity if query.matches else None) - definition_record: DeclarationRecord | None = next( - ( - record - for record in resolver.lookup.index.declarations - if record.path == target_path.as_posix() and record.scope is not ScopeKind.PRIVATE - ), - None, - ) - lexical_path: str = ( - Path(declaration_lexical_path(record=definition_record)).as_posix() - if definition_record is not None - else target_path.as_posix() - ) - return ( - ResourceRecord( - identity=ResourceIdentity(ResourceKind.MODEL, f""), - path=lexical_path, - ownership_root=OwnershipRoot(path=CURRENT_PATH_COMPONENT), - ), - ), None - - -def _rebind_declaration_context( - *, context: DeclarationResolutionContext, consumer: ResourceIdentity -) -> DeclarationResolutionContext: - return replace( - context, - consumer=consumer, - enums=dict(context.enums), - constants=dict(context.constants), - macros=dict(context.macros), - macro_records=dict(context.macro_records), - inaccessible_enums=dict(context.inaccessible_enums), - inaccessible_constants=dict(context.inaccessible_constants), - inaccessible_macros=dict(context.inaccessible_macros), - enum_visibility=_rebind_visibility( - records_by_name=context.enum_visibility, consumer=consumer - ), - constant_visibility=_rebind_visibility( - records_by_name=context.constant_visibility, consumer=consumer - ), - macro_visibility=_rebind_visibility( - records_by_name=context.macro_visibility, consumer=consumer - ), - ) - - -def _rebind_visibility( - *, records_by_name: dict[str, tuple[VisibilityRecord, ...]], consumer: ResourceIdentity -) -> dict[str, tuple[VisibilityRecord, ...]]: - result: dict[str, tuple[VisibilityRecord, ...]] = {} - for name, records in records_by_name.items(): - result[name] = tuple(replace(record, resource=consumer) for record in records) - return result - - -def declaration_usage_records( - *, sql: str, resource: ResourceIdentity, declarations: DeclarationResolutionContext -) -> tuple[UsageRecord, ...]: - """Collect declaration usages with path or expected-model provenance.""" - - scanned, stop = scan_native_declaration_references(sqls=(sql,), skip_malformed=True)[0] - if stop is not None: - raise CompileInputError(_unclosed_text_message(stop)) - usages: list[UsageRecord] = [] - for kind, name, _member, _start, _end in scanned: - visibility: tuple[VisibilityRecord, ...] = ( - declarations.enum_visibility.get(name, ()) - if kind == ENUM_REFERENCE_KIND_CODE - else declarations.constant_visibility.get(name, ()) - ) - usages.extend( - UsageRecord( - consumer=resource, - declaration=record.declaration, - kind=UsageKind.RUNTIME, - through=record.through, - ) - for record in usage_visibility(visibility=visibility, consumer=resource) - ) - return tuple(dict.fromkeys(usages)) - - -def resolve_declaration_expansion( - *, - context: DeclarationExpansionContext, - file_path: Path, - resource: ResourceIdentity | None = None, -) -> DeclarationExpansionContext: - """Return an expansion context scoped to one authored resource path.""" - - if context.resolver is None or not context.resolver.lookup.index.declarations: - return context - return replace( - context, - declarations=resolve_declaration_context( - resolver=context.resolver, - file_path=file_path, - resource=resource, - ), - ) - - def build_model_declaration_indexes( *, model_file: DiscoveredSqlModelFile ) -> tuple[dict[str, EnumDeclaration], dict[str, ConstantDeclaration]]: @@ -450,28 +270,6 @@ def _validate_no_inherited_column_overrides( ) -def declaration_reference_usages( - *, - declarations: DeclarationResolutionContext, - visibility: tuple[VisibilityRecord, ...], - enum_member: str | None, -) -> tuple[UsageRecord, ...]: - """Usage records one resolved `@enum`/`@const` reference adds for the context's consumer.""" - - consumer: ResourceIdentity | DeclarationIdentity | None = declarations.consumer - if consumer is None: - return () - return tuple( - UsageRecord( - consumer=consumer, - declaration=visible.declaration, - through=visible.through, - enum_member=enum_member, - ) - for visible in usage_visibility(visibility=visibility, consumer=consumer) - ) - - def expand_scanned_declaration_references( # noqa: PLR0913 *, sql: str, @@ -486,124 +284,31 @@ def expand_scanned_declaration_references( # noqa: PLR0913 scanned, stop = references if not scanned and stop is None: return DeclarationExpansionResult(sql=sql, spans=(), usages=()) - parts: list[str] = [] - spans: list[ExpansionSpan] = [] - usages: list[UsageRecord] = [] - cursor: int = 0 - output_length: int = 0 - for kind, name, member, start, end in scanned: - replacement: str - visibility: tuple[VisibilityRecord, ...] - if kind == ENUM_REFERENCE_KIND_CODE: - replacement = _enum_member_text( - name=name, - member_name=cast(str, member), - file_path=file_path, - enums=declarations.enums, - inaccessible_enums=declarations.inaccessible_enums, - ) - visibility = declarations.enum_visibility.get(name, ()) - else: - replacement = _constant_text( - name=name, - file_path=file_path, - constants=declarations.constants, - inaccessible_constants=declarations.inaccessible_constants, + consumer: ResourceIdentity | DeclarationIdentity | None = declarations.consumer + expanded_sql, spans, usages = _native.expand_declaration_references( + (sql, list(scanned), stop), + declarations, + ( + consumer, + isinstance(consumer, ResourceIdentity) + and consumer.kind in {ResourceKind.TEST, ResourceKind.SCENARIO}, + str(file_path), + ), + ( + partial( + render_constant_declaration, value_renderer=value_renderer, collection_rendering=collection_rendering, - ) - visibility = declarations.constant_visibility.get(name, ()) - parts.append(sql[cursor:start]) - output_length += start - cursor - usages.extend( - declaration_reference_usages( - declarations=declarations, visibility=visibility, enum_member=member - ) - ) - parts.append(replacement) - spans.append( - ExpansionSpan( - source_start=start, - source_end=end, - output_start=output_length, - output_end=output_length + len(replacement), - ) - ) - output_length += len(replacement) - cursor = end - if stop is not None: - raise CompileInputError(_stop_message(stop=stop, file_path=file_path)) - parts.append(sql[cursor:]) - return DeclarationExpansionResult( - sql="".join(parts), spans=tuple(spans), usages=tuple(dict.fromkeys(usages)) + file_path=file_path, + ), + {"usage_record": UsageRecord, "compile_error": CompileInputError}, + ), ) - - -def _stop_message(*, stop: int, file_path: Path) -> str: - if stop == INVALID_ENUM_REFERENCE_STOP_CODE: - return _invalid_reference_message(kind=_ENUM_REFERENCE_KIND, file_path=file_path) - if stop == INVALID_CONSTANT_REFERENCE_STOP_CODE: - return _invalid_reference_message(kind=_CONSTANT_REFERENCE_KIND, file_path=file_path) - return _unclosed_text_message(stop) - - -def _unclosed_text_message(stop: int) -> str: - if stop == UNCLOSED_BLOCK_COMMENT_STOP_CODE: - return f"{_CONTEXT} contains an unclosed block comment" - return f"{_CONTEXT} contains an unclosed quoted string" - - -def _invalid_reference_message(*, kind: str, file_path: Path) -> str: - noun: str = "enum" if kind == _ENUM_REFERENCE_KIND else "constant" - return f"Invalid {noun} reference in '{file_path}'" - - -def usage_visibility( - *, - visibility: tuple[VisibilityRecord, ...], - consumer: ResourceIdentity | DeclarationIdentity, -) -> tuple[VisibilityRecord, ...]: - if not isinstance(consumer, ResourceIdentity) or consumer.kind not in { - ResourceKind.TEST, - ResourceKind.SCENARIO, - }: - return visibility - relationship_visibility: tuple[VisibilityRecord, ...] = tuple( - record for record in visibility if record.through is not None + return DeclarationExpansionResult( + sql=expanded_sql, + spans=tuple(ExpansionSpan(*span) for span in spans), + usages=tuple(dict.fromkeys(usages)), ) - return relationship_visibility or visibility - - -def resolve_enum_contract_columns( - *, - schema_entry: SchemaModelEntry | None, - config_values: dict[str, object], - enums: dict[str, EnumDeclaration], -) -> tuple[SchemaModelEntry | None, dict[str, EnumDeclaration]]: - """Resolve enum column types and synthesize enforced accepted-values audits.""" - - if schema_entry is None: - return None, {} - contract_enforced: bool = config_values.get("contract") == ContractPolicy.ENFORCED - enum_columns: dict[str, EnumDeclaration] = {} - columns: list[SchemaColumn] = [] - column: SchemaColumn - for column in schema_entry.columns: - declaration: EnumDeclaration | None = enums.get(column.type or "") - if declaration is None: - columns.append(column) - continue - enum_columns[column.name] = declaration - audits: tuple[SchemaAuditInstance, ...] = column.audits - if contract_enforced: - generated_audit: SchemaAuditInstance = SchemaAuditInstance( - definition_name=_ACCEPTED_VALUES_AUDIT, - arguments={"values": tuple(member.value for member in declaration.members)}, - ) - if generated_audit not in audits: - audits = (*audits, generated_audit) - columns.append(replace(column, type=declaration.scalar_type, audits=audits)) - return replace(schema_entry, columns=tuple(columns)), enum_columns def _with_declaration[T: EnumDeclaration | ConstantDeclaration]( @@ -618,72 +323,6 @@ def _with_declaration[T: EnumDeclaration | ConstantDeclaration]( return declarations | {declaration.name: declaration} -def _enum_member_text( - *, - name: str, - member_name: str, - file_path: Path, - enums: dict[str, EnumDeclaration], - inaccessible_enums: dict[str, DeclarationRecord], -) -> str: - declaration: EnumDeclaration | None = enums.get(name) - if declaration is None: - inaccessible: DeclarationRecord | None = inaccessible_enums.get(name) - if inaccessible is not None: - raise CompileInputError( - _inaccessible_declaration_message( - kind="enum", name=name, record=inaccessible, consumer=file_path - ), - ) - scope_help: str = " in this model" if name.startswith("_") else "" - visible: str = ", ".join(sorted(enums)) or "none" - raise CompileInputError( - f"Unknown enum '{name}'{scope_help} in '{file_path}'. Visible enums: {visible}", - ) - member: EnumMember | None = next( - (candidate for candidate in declaration.members if candidate.name == member_name), - None, - ) - if member is None: - available: str = ", ".join(item.name for item in declaration.members) - raise CompileInputError( - f"Unknown member '{member_name}' for enum '{name}' in '{file_path}'. " - f"Available members: {available}", - ) - return render_enum_member_value(value=member.value) - - -def _constant_text( - *, - name: str, - file_path: Path, - constants: dict[str, ConstantDeclaration], - inaccessible_constants: dict[str, DeclarationRecord], - value_renderer: TypedSqlValueRenderer, - collection_rendering: CollectionRendering, -) -> str: - declaration: ConstantDeclaration | None = constants.get(name) - if declaration is None: - inaccessible: DeclarationRecord | None = inaccessible_constants.get(name) - if inaccessible is not None: - raise CompileInputError( - _inaccessible_declaration_message( - kind="constant", name=name, record=inaccessible, consumer=file_path - ), - ) - scope_help: str = " in this model" if name.startswith("_") else "" - visible: str = ", ".join(sorted(constants)) or "none" - raise CompileInputError( - f"Unknown constant '{name}'{scope_help} in '{file_path}'. Visible constants: {visible}", - ) - return render_constant_declaration( - declaration=declaration, - value_renderer=value_renderer, - collection_rendering=collection_rendering, - file_path=file_path, - ) - - def render_constant_declaration( *, declaration: ConstantDeclaration, @@ -726,23 +365,6 @@ def render_constant_declaration( return rendered -def _inaccessible_declaration_message( - *, kind: str, name: str, record: DeclarationRecord, consumer: Path -) -> str: - owner: str = record.owning_path or record.ownership_root.path - owner_identity: str = ( - f" ({record.identity.owner.kind.value} '{record.identity.owner.name}')" - if record.identity.owner is not None - else "" - ) - return ( - f"{kind.capitalize()} '{name}' is known but inaccessible in '{consumer}'. " - f"It is defined at {record.path}:{record.line}:{record.column} with " - f"{record.scope.value} scope owned by '{owner}'{owner_identity}; " - f"consumer path: '{consumer}'" - ) - - def render_enum_member_value(*, value: str | int) -> str: if isinstance(value, int): return str(value) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py b/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py deleted file mode 100644 index 856cda760a..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/macro_argument_values.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Build macro call arguments from the value plan native parsing returns for their text.""" - -from __future__ import annotations - -from collections.abc import Callable, Sequence -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( - typed_reference_value, -) -from sqlbuild.compiler.compile.constants import ( - MACRO_ARGUMENT_ERROR_TAG, - MACRO_ARGUMENT_NESTED_CALL_TAG, -) -from sqlbuild.compiler.compile.exceptions import MacroArgumentError -from sqlbuild.compiler.compile.models import ParsedMacroArguments - -type _ValueRow = tuple[object, ...] -type _BuildContext = tuple[Sequence[object], Callable[..., MacroArgumentError]] -_BUILDERS: dict[str, Callable[..., object]] = { - "s": lambda *, row, context: row[1], - "b": lambda *, row, context: row[1], - "n": lambda *, row, context: None, - "i": lambda *, row, context: int(cast(str, row[2]), cast(int, row[1])), - "f": lambda *, row, context: float(cast(str, row[1])), - "c": lambda *, row, context: context[0][cast(int, row[1])], - "r": lambda *, row, context: typed_reference_value( - function=cast(str, row[1]), name=cast(str, row[2]) - ), - "l": lambda *, row, context: _items(row=row, context=context), - "t": lambda *, row, context: tuple(_items(row=row, context=context)), - "d": lambda *, row, context: _mapping(row=row, context=context), - "-": lambda *, row, context: -_number(row=row, context=context, sign="-"), - "+": lambda *, row, context: _number(row=row, context=context, sign="+"), -} - - -def parse_macro_call_arguments( - *, - args_source: str, - nested: tuple[tuple[int, int], ...], - nested_values: Sequence[object], - macro_name: str, - file_path: Path, -) -> ParsedMacroArguments: - """Parse `args_source` natively; nested calls at `nested` take their `nested_values`.""" - - parsed: tuple[object, ...] = _native.parse_macro_arguments(args_source, list(nested)) - - def argument_error(*, offset: int, detail: str, help_text: str) -> MacroArgumentError: - line: int = args_source.count("\n", 0, offset) + 1 - column: int = offset - (args_source.rfind("\n", 0, offset) + 1) + 1 - return MacroArgumentError( - detail=detail, - help=help_text, - macro_name=macro_name, - file_path=file_path, - offset=offset, - relative_position=(line, column), - ) - - if parsed[0] == MACRO_ARGUMENT_ERROR_TAG: - _, detail, help_text, line, column = parsed - line_start: int = sum( - len(text) + 1 for text in args_source.split("\n")[: cast(int, line) - 1] - ) - raise argument_error( - offset=line_start + cast(int, column) - 1, - detail=cast(str, detail), - help_text=cast(str, help_text), - ) - _, positional, keywords, references = parsed - - def unary_error(*, call: int, sign: str) -> MacroArgumentError: - value: object = nested_values[call] - return argument_error( - offset=nested[call][0], - detail=( - f"apply unary {sign} to the value of a nested macro call, which is a " - f"{type(value).__name__} and not a number" - ), - help_text="Return a number from the nested macro, or apply the sign inside the macro", - ) - - context: _BuildContext = (nested_values, unary_error) - return ParsedMacroArguments( - args=tuple(_built(row=row, context=context) for row in cast(list[_ValueRow], positional)), - kwargs={ - name: _built(row=row, context=context) - for name, row in cast(list[tuple[str, _ValueRow]], keywords) - }, - typed_references=tuple( - typed_reference_value(function=function, name=name) - for function, name in cast(list[tuple[str, str]], references) - ), - ) - - -def _built(*, row: _ValueRow, context: _BuildContext) -> object: - return _BUILDERS[cast(str, row[0])](row=row, context=context) - - -def _items(*, row: _ValueRow, context: _BuildContext) -> list[object]: - return [_built(row=item, context=context) for item in cast(list[_ValueRow], row[1])] - - -def _mapping(*, row: _ValueRow, context: _BuildContext) -> dict[object, object]: - return { - _built(row=key, context=context): _built(row=value, context=context) - for key, value in cast(list[tuple[_ValueRow, _ValueRow]], row[1]) - } - - -def _number(*, row: _ValueRow, context: _BuildContext, sign: str) -> int | float: - operand: _ValueRow = cast(_ValueRow, row[1]) - number: object = _built(row=operand, context=context) - if not isinstance(number, int | float): - raise context[1](call=_nested_call(operand), sign=sign) - return number - - -def _nested_call(row: _ValueRow) -> int: - """The nested call index under unary signs; only a nested call yields a non-number here.""" - - while row[0] != MACRO_ARGUMENT_NESTED_CALL_TAG: - row = cast(_ValueRow, row[1]) - return cast(int, row[1]) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index b039839358..56605a2eb4 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -9,25 +9,21 @@ import sys import threading from collections.abc import Callable, Iterator, Mapping -from contextvars import Token from dataclasses import dataclass, replace +from functools import cache, partial from pathlib import Path from types import MappingProxyType, ModuleType -from typing import NoReturn, cast +from typing import cast import sqlbuild._native as _native +from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( call_site_sql_references, - reject_macro_generated_references, - report_macro_reference_call_syntax, + typed_reference_kinds, ) -from sqlbuild.compiler.compile._helpers.render.macro_argument_values import ( - parse_macro_call_arguments, -) -from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts from sqlbuild.compiler.compile.constants import ( DECLARATION_REFERENCE_NAMES, - MACRO_CONTEXT_PARAMETER_NAME, + MACRO_TOKEN, ) from sqlbuild.compiler.compile.exceptions import ( CompileInputError, @@ -35,33 +31,28 @@ MacroDeclarationLookupError, ) from sqlbuild.compiler.compile.models import ( + CompilerDiagnostic, DeclarationResolutionContext, DeclarationScopeResolver, ExpansionSpan, LoadedMacro, MacroContext, MacroExpansionResult, - MacroExpansionState, - ParsedMacroArguments, StaticMacroExport, StaticMacroFault, StaticMacroInventory, ) +from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, DiscoveredMacroFile, EnumDeclaration, ) -from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ( - DECLARATION_READ_EVENT, GENERATED_REFERENCE_MARKERS, - GENERATED_SQL_EVENT, - MACRO_USE_EVENT, ) -from sqlbuild.compiler.macro_bridge.main._scan_macro_calls import scan_macro_calls +from sqlbuild.compiler.macro_bridge.main._macro_store_tokens import macro_store_tokens from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge -from sqlbuild.compiler.macro_bridge.models import MacroCallScan, MacroCallSite, MacroScanFailure from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( validate_resource_identity, ) @@ -70,10 +61,12 @@ DeclarationRecord, ResourceIdentity, UsageRecord, - VisibilityRecord, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind, UsageKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.python_nodes.models import RELATION_RENDERER, SqlResourceRef +from sqlbuild.python_nodes.types import SqlResourceRefKind +from sqlbuild.spec.contracts.models import SourceLocation from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind @@ -1175,42 +1168,95 @@ def expand_sql_macros_result( ) -> MacroExpansionResult: """Expand macros and retain deterministic resolved dependency facts.""" - facts: MacroExpansionFacts = MacroExpansionFacts() - state: MacroExpansionState = MacroExpansionState( - loaded_macros=loaded_macros, - macro_overrides={} if macro_overrides is None else macro_overrides, - macro_context=macro_context, - declaration_resolver=declaration_resolver, - declarations=declarations, - facts=facts, - consumer=consumer, + if MACRO_TOKEN not in sql: + return MacroExpansionResult(sql=sql) + if declarations is None and declaration_resolver is not None: + from sqlbuild.compiler.compile._helpers.render.declarations import ( + resolve_declaration_context, + ) + + declarations = resolve_declaration_context( + resolver=declaration_resolver, file_path=file_path + ) + expansion: _native.NativeMacroExpansion = _native.NativeMacroExpansion( + file_path, + ( + loaded_macros, + {} if macro_overrides is None else macro_overrides, + macro_context, + declarations, + consumer, + ), + macro_context._enforce_explicit_references, + macro_expansion_classes(), ) - bridge: MacroBridge | None = active_macro_bridge() - expanded_sql, spans = _bridged_sql_macros( - sql=sql, consumer_path=file_path, state=state, bridge=bridge + expanded_sql, spans, argument_references, dependencies, usages = expansion.expand( + sql, active_macro_bridge() ) return MacroExpansionResult( sql=expanded_sql, - spans=spans, - argument_references=call_site_sql_references(tuple(facts.argument_references)), - dependencies=tuple(dict.fromkeys(facts.dependencies)), - usages=tuple(dict.fromkeys(facts.usages)), + spans=tuple( + ExpansionSpan( + source_start=source_start, + source_end=source_end, + output_start=output_start, + output_end=output_end, + ) + for source_start, source_end, output_start, output_end in spans + ), + argument_references=call_site_sql_references( + tuple( + SqlResourceRef(kind=SqlResourceRefKind(kind), name=name) + for kind, name in argument_references + ) + ), + dependencies=tuple(dict.fromkeys(dependencies)), + usages=tuple(dict.fromkeys(usages)), ) -def _bridged_sql_macros( - *, sql: str, consumer_path: Path, state: MacroExpansionState, bridge: MacroBridge | None -) -> tuple[str, tuple[ExpansionSpan, ...]]: - from sqlbuild.compiler.compile._helpers.macro_bridge.expansion import ( - expand_bridged_sql_macros, - expand_unbridged_sql_macros, - ) - - if bridge is None: - return expand_unbridged_sql_macros(sql=sql, consumer_path=consumer_path, state=state) - return expand_bridged_sql_macros( - sql=sql, consumer_path=consumer_path, state=state, bridge=bridge - ) +@cache +def macro_expansion_classes() -> dict[str, object]: + """The classes, enums and helpers the native expansion builds facts and errors from.""" + + from sqlbuild.compiler.frontier.exceptions import NativeStageMismatchError + from sqlbuild.compiler.frontier.main._report_native_fallback import report_native_fallback + from sqlbuild.compiler.frontier.types import NativeFallbackSite + + return { + "compile_error": CompileInputError, + "macro_argument_error": MacroArgumentError, + "mismatch_error": NativeStageMismatchError, + "declaration_identity": DeclarationIdentity, + "declaration_kind": DeclarationKind, + "macro_kind": DeclarationKind.MACRO, + "usage_record": UsageRecord, + "runtime_usage": UsageKind.RUNTIME, + "resource_identity": ResourceIdentity, + "model_kind": ResourceKind.MODEL, + "resource_ref": SqlResourceRef, + "ref_kinds": typed_reference_kinds(), + "ref_functions": { + kind.value: function for function, kind in typed_reference_kinds().items() + }, + "store_tokens": macro_store_tokens(), + "relation_renderer": RELATION_RENDERER, + "invocation_context": macro_invocation_context, + "generated": { + "compiler_diagnostic": CompilerDiagnostic, + "report_diagnostic": report_compile_diagnostic, + "source_location": SourceLocation, + "resource_identity": ResourceIdentity, + "generic_syntax": SqlLexicalSyntax().native_mapping, + "generated_help": explicit_references_help(allowed="macro-generated references"), + "all_markers": list(GENERATED_REFERENCE_MARKERS), + }, + "report_mocked": partial( + report_native_fallback, site=NativeFallbackSite.MACRO_CALL_MOCKED, kind="test_mock" + ), + "int_type": int, + "float_type": float, + } def _inventory_project_macros( @@ -1227,21 +1273,6 @@ def _inventory_project_macros( return _inventory_from_analyses(macro_files=macro_files, analyses=analyses) -def _expansion_declarations( - *, state: MacroExpansionState, consumer_path: Path -) -> DeclarationResolutionContext | None: - declarations: DeclarationResolutionContext | None = state.declarations - if declarations is None and state.declaration_resolver is not None: - from sqlbuild.compiler.compile._helpers.render.declarations import ( - resolve_declaration_context, - ) - - declarations = resolve_declaration_context( - resolver=state.declaration_resolver, file_path=consumer_path - ) - return declarations - - def find_macro_call_names(sql: str) -> tuple[str, ...]: """Return unique authored macro call names in encounter order.""" @@ -1252,16 +1283,6 @@ def find_macro_call_names(sql: str) -> tuple[str, ...]: ) -def find_nested_macro_call_names(sql: str) -> tuple[str, ...]: - """Return unique macro call names, including calls inside other calls' arguments.""" - - return tuple( - name - for name in _native_macro_call_names(sql=sql, nested=True) - if name not in DECLARATION_REFERENCE_NAMES - ) - - def _native_macro_call_names(*, sql: str, nested: bool) -> tuple[str, ...]: """Every call name Python's scan finds, declaration calls included.""" @@ -1273,263 +1294,36 @@ def _native_macro_call_names(*, sql: str, nested: bool) -> tuple[str, ...]: return tuple(names) -def raise_failed_macro_call( - *, - failure: MacroScanFailure, - file_path: Path, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, -) -> NoReturn: - """Raise where Python's scan raises: the failing call's macro lookup error, else the scan's.""" - - if failure.call_name is not None: - _ = _resolved_macro( - macro_name=failure.call_name, - file_path=file_path, - state=state, - declarations=declarations, - ) - raise CompileInputError(failure.message) - - -def _resolved_macro( +def macro_invocation_context( *, - macro_name: str, - file_path: Path, - state: MacroExpansionState, + expansion: _native.NativeMacroExpansion, declarations: DeclarationResolutionContext | None, -) -> LoadedMacro: - loaded_macro: LoadedMacro | None = ( - declarations.macros.get(macro_name) - if declarations is not None - else state.loaded_macros.get(macro_name) - ) - if declarations is not None and macro_name in declarations.inaccessible_macros: - record: DeclarationRecord = declarations.inaccessible_macros[macro_name] - suggestions: str = ", ".join(sorted(declarations.macros)) or "none" - owner: str = record.owning_path or "global" - raise CompileInputError( - f"Macro '@{macro_name}' in '{file_path}' is inaccessible. Defined at " - f"'{record.path}:{record.line}:{record.column}' with scope owner '{owner}'; " - f"consumer/definition path is '{file_path}'. Visible macros: {suggestions}" - ) - if loaded_macro is None: - available: object = declarations.macros if declarations is not None else state.loaded_macros - available_macro_names: str = ", ".join(sorted(available)) or "none" - raise CompileInputError( - f"Unknown macro '@{macro_name}' in '{file_path}'. Available macros: " - f"{available_macro_names}" - ) - return loaded_macro - - -def evaluate_macro_call( - *, - sql: str, - site: MacroCallSite, file_path: Path, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, - stack: tuple[DeclarationIdentity, ...], - top_level: bool, -) -> object: - """Run one natively scanned call: its nested calls, then the macro, in Python's order.""" - - macro_name: str = site.name - loaded_macro: LoadedMacro = _resolved_macro( - macro_name=macro_name, file_path=file_path, state=state, declarations=declarations - ) - override_value: str | None = state.macro_overrides.get(macro_name) - if override_value is not None: - _validate_final_macro_sql( - macro_name=macro_name, file_path=file_path, macro_result=override_value - ) - return override_value - identity: DeclarationIdentity = ( - declarations.macro_records[macro_name].identity - if declarations is not None - else DeclarationIdentity(kind=DeclarationKind.MACRO, name=macro_name) - ) - if identity in stack: - cycle: tuple[DeclarationIdentity, ...] = (*stack[stack.index(identity) :], identity) - chain: str = " -> ".join(item.name for item in cycle) - paths: str = " -> ".join( - str(state.loaded_macros[item.name].relative_path) for item in cycle - ) - raise CompileInputError( - f"Macro expansion cycle detected: {chain}. Definition paths: {paths}" - ) - _record_macro_use( - macro_name=macro_name, - identity=identity, - state=state, - declarations=declarations, - stack=stack, - ) - args_source: str = sql[site.args_start : site.end - 1] - args: tuple[object, ...] - kwargs: dict[str, object] - _ = state.facts.open_call_site() - try: - args, kwargs = _parse_macro_arguments( - macro_name=macro_name, - args_source=args_source, - file_path=file_path, - state=state, - declarations=declarations, - stack=stack, - ) - except MacroArgumentError as error: - raise error.shifted(site.args_start) from None - finally: - approved: frozenset[SqlResourceRef] = state.facts.close_call_site() - try: - invocation_context: MacroContext = _build_macro_invocation_context( - macro_context=state.macro_context, - declarations=declarations, - file_path=file_path, - state=state, - ) - renderer_token: Token[Callable[[object], str] | None] = RELATION_RENDERER.set( - state.facts.begin_rendering(approved).render_relation - ) - try: - macro_result: object = _call_loaded_macro( - loaded_macro=loaded_macro, - macro_context=invocation_context, - args=args, - kwargs=kwargs, - ) - finally: - RELATION_RENDERER.reset(renderer_token) - _ = state.facts.end_rendering() - except CompileInputError: - raise - except TypeError as error: - raise CompileInputError( - f"Macro '@{macro_name}' in '{file_path}' could not be called: {error}" - ) from error - except Exception as error: - raise CompileInputError( - f"Macro '@{macro_name}' in '{file_path}' failed: {error}" - ) from error - if top_level and not isinstance(macro_result, str): - raise CompileInputError( - f"Macro '@{macro_name}' in '{file_path}' must return a SQL string when " - "used directly in SQL" - ) - if isinstance(macro_result, str): - _validate_final_macro_sql( - macro_name=macro_name, file_path=file_path, macro_result=macro_result - ) - _report_generated_references( - macro_name=macro_name, - loaded_macro=loaded_macro, - macro_result=macro_result, - file_path=file_path, - state=state, - ) - return macro_result - - -def _record_macro_use( - *, - macro_name: str, - identity: DeclarationIdentity, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, - stack: tuple[DeclarationIdentity, ...], -) -> None: - state.facts.note_event((MACRO_USE_EVENT, macro_name, "")) - _ = state.facts.add_dependency(identity) - if stack: - state.facts.block_memo() - _ = state.facts.add_usage( - UsageRecord(stack[-1], identity, UsageKind.DECLARATION_DEPENDENCY), - ) - elif state.consumer is not None: - visibility: tuple[VisibilityRecord, ...] = ( - declarations.macro_visibility.get(macro_name, ()) if declarations is not None else () - ) - if visibility: - from sqlbuild.compiler.compile._helpers.render.declarations import usage_visibility - - for visible in usage_visibility(visibility=visibility, consumer=state.consumer): - _ = state.facts.add_usage( - UsageRecord( - state.consumer, - identity, - UsageKind.RUNTIME, - through=visible.through, - ) - ) - else: - _ = state.facts.add_usage(UsageRecord(state.consumer, identity, UsageKind.RUNTIME)) - - -def _report_generated_references( - *, - macro_name: str, - loaded_macro: LoadedMacro, - macro_result: str, - file_path: Path, - state: MacroExpansionState, -) -> None: - if any(marker in macro_result for marker in GENERATED_REFERENCE_MARKERS): - state.facts.note_event((GENERATED_SQL_EVENT, macro_name, macro_result)) - report_macro_reference_call_syntax( - loaded_macro=loaded_macro, - macro_result=macro_result, - file_path=file_path, - consumer=state.consumer, - ) - if state.macro_context._enforce_explicit_references: - reject_macro_generated_references( - loaded_macro=loaded_macro, - macro_result=macro_result, - file_path=file_path, - consumer=state.consumer, - ) - - -def _build_macro_invocation_context( - *, macro_context: MacroContext, - declarations: DeclarationResolutionContext | None, - file_path: Path, - state: MacroExpansionState, ) -> MacroContext: + """The context a macro is called with: visible constants and enums that record their reads.""" + if declarations is None: return macro_context constant_values: Mapping[str, object] = _build_constant_context_values(declarations.constants) enum_values: Mapping[str, object] = _build_enum_context_values( - declarations=declarations.enums, file_path=file_path, on_miss=state.facts.block_memo + declarations=declarations.enums, file_path=file_path, on_miss=expansion.block_memo ) constants: Mapping[str, object] = _MacroDeclarationValues( values=constant_values, inaccessible=declarations.inaccessible_constants, declaration_kind="constant", file_path=file_path, - on_access=lambda name: _record_macro_declaration_usage( - name=name, - kind=DeclarationKind.CONSTANT, - declarations=declarations, - state=state, - ), - on_miss=state.facts.block_memo, + on_access=partial(expansion.declaration_read, DeclarationKind.CONSTANT.value), + on_miss=expansion.block_memo, ) enum_mapping: Mapping[str, object] = _MacroDeclarationValues( values=enum_values, inaccessible=declarations.inaccessible_enums, declaration_kind="enum", file_path=file_path, - on_access=lambda name: _record_macro_declaration_usage( - name=name, - kind=DeclarationKind.ENUM, - declarations=declarations, - state=state, - ), - on_miss=state.facts.block_memo, + on_access=partial(expansion.declaration_read, DeclarationKind.ENUM.value), + on_miss=expansion.block_memo, ) return replace( macro_context, @@ -1605,134 +1399,3 @@ def _constant_object_items(value: SqlValue) -> tuple[tuple[str, SqlValue], ...]: ): raise CompileInputError("Invalid normalized object constant in macro context") return cast(tuple[tuple[str, SqlValue], ...], value.value) - - -def _record_macro_declaration_usage( - *, - name: str, - kind: DeclarationKind, - declarations: DeclarationResolutionContext, - state: MacroExpansionState, -) -> None: - from sqlbuild.compiler.compile._helpers.render.declarations import usage_visibility - - state.facts.note_event((DECLARATION_READ_EVENT, kind.value, name)) - consumer: ResourceIdentity | DeclarationIdentity | None = ( - state.consumer or declarations.consumer - ) - if consumer is None: - return - visibility: tuple[VisibilityRecord, ...] = ( - declarations.constant_visibility.get(name, ()) - if kind == DeclarationKind.CONSTANT - else declarations.enum_visibility.get(name, ()) - ) - if visibility: - visible_records: tuple[VisibilityRecord, ...] = usage_visibility( - visibility=visibility, - consumer=consumer, - ) - for visible in visible_records: - _ = state.facts.add_usage( - UsageRecord( - consumer=consumer, - declaration=visible.declaration, - kind=UsageKind.RUNTIME, - through=visible.through, - ) - ) - return - else: - declaration: ConstantDeclaration | EnumDeclaration = ( - declarations.constants[name] - if kind == DeclarationKind.CONSTANT - else declarations.enums[name] - ) - owner: ResourceIdentity | None = ( - ResourceIdentity(kind=ResourceKind.MODEL, name=declaration.model_name) - if declaration.model_name is not None - else None - ) - identity: DeclarationIdentity = DeclarationIdentity(kind=kind, name=name, owner=owner) - _ = state.facts.add_usage(UsageRecord(consumer, identity, UsageKind.RUNTIME)) - - -def _validate_final_macro_sql(*, macro_name: str, file_path: Path, macro_result: str) -> None: - try: - _ = macro_result.encode("utf-8") - except UnicodeEncodeError as error: - raise CompileInputError( - f"Macro '@{macro_name}' in '{file_path}' returned text with the lone surrogate " - f"{macro_result[error.start]!r}, which is not valid Unicode", - help=( - "Return valid Unicode text from the macro; lone surrogates usually come from " - "bytes decoded with errors='surrogateescape'" - ), - ) from None - generated_calls: tuple[str, ...] = _native_macro_call_names(sql=macro_result, nested=False) - if not generated_calls: - return - calls: str = ", ".join(f"@{name}()" for name in generated_calls) - raise CompileInputError( - f"Macro '@{macro_name}' in '{file_path}' returned SQLBuild call(s) {calls}. " - "Macro output must be final SQL. Compose macros with ordinary Python imports and " - "function calls instead; nested @macro() calls are supported only when explicitly " - "authored in SQL" - ) - - -def _call_loaded_macro( - *, - loaded_macro: LoadedMacro, - macro_context: MacroContext, - args: tuple[object, ...], - kwargs: dict[str, object], -) -> object: - if loaded_macro.injects_context: - if MACRO_CONTEXT_PARAMETER_NAME in kwargs: - raise CompileInputError( - f"Macro '@{loaded_macro.name}' must not be called with keyword argument 'ctx'; " - "'ctx' is reserved for injected macro context" - ) - return loaded_macro.function(macro_context, *args, **kwargs) - return loaded_macro.function(*args, **kwargs) - - -def _parse_macro_arguments( - *, - macro_name: str, - args_source: str, - file_path: Path, - state: MacroExpansionState, - declarations: DeclarationResolutionContext | None, - stack: tuple[DeclarationIdentity, ...], -) -> tuple[tuple[object, ...], dict[str, object]]: - if not args_source.strip(): - return (), {} - scan: MacroCallScan = scan_macro_calls(sql=args_source) - nested: list[tuple[int, int]] = [(site.start, site.end) for site in scan.sites] - nested_values: list[object] = [ - evaluate_macro_call( - sql=args_source, - site=site, - file_path=file_path, - state=state, - declarations=declarations, - stack=stack, - top_level=False, - ) - for site in scan.sites - ] - if scan.failure is not None: - raise_failed_macro_call( - failure=scan.failure, file_path=file_path, state=state, declarations=declarations - ) - parsed: ParsedMacroArguments = parse_macro_call_arguments( - args_source=args_source, - nested=tuple(nested), - nested_values=nested_values, - macro_name=macro_name, - file_path=file_path, - ) - state.facts.record_call_site_refs(parsed.typed_references) - return parsed.args, parsed.kwargs diff --git a/src/sqlbuild/compiler/compile/_helpers/render/parameters.py b/src/sqlbuild/compiler/compile/_helpers/render/parameters.py deleted file mode 100644 index 4aaaa82799..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/parameters.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Typed parameter expansion for SQL-native test cases.""" - -from __future__ import annotations - -import re -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.sql_values.exceptions import SqlValueRenderingError, SqlValueValidationError -from sqlbuild.sql_values.main.validate_rendered_size import validate_rendered_sql_value_size -from sqlbuild.sql_values.models import SqlValue - -_PARAMETER_REFERENCE: re.Pattern[str] = re.compile( - r'@param(?![A-Za-z0-9_])\s*\(\s*"(?P[A-Za-z_][A-Za-z0-9_]*)"\s*\)' -) -_PARAMETER_TOKEN: re.Pattern[str] = re.compile(r"@param(?![A-Za-z0-9_])") - - -def expand_test_parameters( - *, - sql: str, - file_path: Path, - values: tuple[tuple[str, SqlValue], ...], - value_renderer: TypedSqlValueRenderer, - test_name: str, - case_name: str, -) -> tuple[str, frozenset[str]]: - """Render active parameter references while leaving comments and quoted text unchanged.""" - - value_lookup: dict[str, SqlValue] = dict(values) - references, error = _native.scan_test_parameter_references( - sql, list(value_lookup), f"SQL test '{test_name}' case '{case_name}' in '{file_path}'" - ) - expanded: tuple[str, frozenset[str]] = _splice_parameter_references( - sql=sql, - references=references, - value_lookup=value_lookup, - value_renderer=value_renderer, - test_name=test_name, - case_name=case_name, - ) - if error is not None: - raise CompileInputError(error) - return expanded - - -def _splice_parameter_references( - *, - sql: str, - references: list[tuple[int, int, str]], - value_lookup: dict[str, SqlValue], - value_renderer: TypedSqlValueRenderer, - test_name: str, - case_name: str, -) -> tuple[str, frozenset[str]]: - parts: list[str] = [] - cursor: int = 0 - for start, end, name in references: - parts.append(sql[cursor:start]) - parts.append( - _render_parameter( - name=name, - value=value_lookup[name], - value_renderer=value_renderer, - test_name=test_name, - case_name=case_name, - ) - ) - cursor = end - parts.append(sql[cursor:]) - return "".join(parts), frozenset(name for _start, _end, name in references) - - -def _render_parameter( - *, - name: str, - value: SqlValue, - value_renderer: TypedSqlValueRenderer, - test_name: str, - case_name: str, -) -> str: - try: - rendered: str = value_renderer.render_typed_scalar(value=value) - validate_rendered_sql_value_size( - rendered_sql=rendered, - context=(f"SQL test '{test_name}' case '{case_name}' parameter '{name}'"), - ) - except (SqlValueRenderingError, SqlValueValidationError) as error: - raise CompileInputError( - f"SQL test '{test_name}' case '{case_name}' parameter '{name}' could not " - f"be rendered by adapter '{value_renderer.adapter_name}': {error}" - ) from error - return rendered diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py index e8c9e0eeaa..ccdfcd649b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/templating.py @@ -6,7 +6,7 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, TEMPLATE_OPEN_TOKEN +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.model_config.constants import ENVIRONMENT_READ from sqlbuild.compiler.model_config.main._expand_native_config_templates import ( @@ -23,8 +23,17 @@ def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: """Resolve merged effective vars with recursive `${name}` expansion.""" - for name, value in raw_values.items(): - _reject_lone_surrogates(name=name, value=value, path="its value") + site: tuple[str, str, int] | None = _native.lone_surrogate_site(raw_values) + if site is not None: + name, path, byte_offset = site + raise CompileInputError( + f"Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte " + f"{byte_offset}, which is not valid Unicode text", + help=( + f"Set '{name}' to valid Unicode text in --vars or the [vars] table; the " + "value is not shown because it may be a secret" + ), + ) outcome: tuple[object, list[tuple[str, str]]] = _native.expand_effective_vars( raw_values, UnicodeEnvironment() ) @@ -34,41 +43,6 @@ def expand_effective_vars(raw_values: dict[str, object]) -> dict[str, object]: return cast(dict[str, object], outcome[0]) -def _reject_lone_surrogates(*, name: str, value: object, path: str) -> None: - if isinstance(value, dict): - for key, item in value.items(): - _reject_lone_surrogates(name=name, value=key, path=f"{path} key") - _reject_lone_surrogates(name=name, value=item, path=f"{path}[{key!a}]") - elif isinstance(value, list | tuple): - for index, item in enumerate(value): - _reject_lone_surrogates(name=name, value=item, path=f"{path}[{index}]") - elif isinstance(value, str): - try: - _ = value.encode("utf-8") - except UnicodeEncodeError as error: - byte_offset: int = len(value[: error.start].encode("utf-8")) - raise CompileInputError( - f"Variable '{name}' holds a lone surrogate in {path} at UTF-8 byte " - f"{byte_offset}, which is not valid Unicode text", - help=( - f"Set '{name}' to valid Unicode text in --vars or the [vars] table; the " - "value is not shown because it may be a secret" - ), - ) from None - - -def contains_template_data(value: object) -> bool: - """Return whether a supported nested value contains a template token.""" - - if isinstance(value, str): - return TEMPLATE_OPEN_TOKEN in value - if isinstance(value, dict): - return any(contains_template_data(item) for item in value.values()) - if isinstance(value, list | tuple): - return any(contains_template_data(item) for item in value) - return False - - def expand_template_data( *, value: object, diff --git a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py b/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py deleted file mode 100644 index fb37e844cc..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py +++ /dev/null @@ -1,55 +0,0 @@ -"""SQL-native scenario compile-semantic extraction helpers.""" - -from __future__ import annotations - -from typing import Any - -import orjson - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.sql_tests.core import scanned_expected_model_names -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileSqlScenarioCte, - CompileSqlScenarioCtes, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def extract_sql_scenario_ctes( - *, sql: str, file_label: str, syntax: SqlLexicalSyntax -) -> CompileSqlScenarioCtes: - """Extract top-level SQL-native scenario fixture, expected, and assertion CTEs.""" - - outcome: dict[str, Any] = orjson.loads( - _native.extract_sql_scenario_json(sql, file_label, syntax.native_mapping) - ) - error: str | None = outcome.get("error") - if error is not None: - raise CompileInputError(error) - payload: dict[str, list[Any]] = outcome["scenario"] - return CompileSqlScenarioCtes( - authored_ctes=_scenario_ctes(payload["authored"]), - expected_ctes=_scenario_ctes(payload["expected"]), - assertion_ctes=_scenario_ctes(payload["assertions"]), - source_fixture_names=tuple(payload["sourceFixtures"]), - ref_fixture_names=tuple(payload["refFixtures"]), - seed_fixture_names=tuple(payload["seedFixtures"]), - dbt_ref_fixture_names=tuple(payload["dbtRefFixtures"]), - expected_model_names=tuple(payload["expectedModels"]), - assertion_names=tuple(payload["assertionNames"]), - ) - - -def _scenario_ctes(values: list[list[str]]) -> tuple[CompileSqlScenarioCte, ...]: - return tuple(CompileSqlScenarioCte(name=name, sql_body=body) for name, body in values) - - -def extract_sql_scenario_expected_model_names( - *, sql: str, file_label: str, syntax: SqlLexicalSyntax -) -> tuple[str, ...]: - """Extract explicit expected-model relationships without inspecting CTE bodies.""" - - return scanned_expected_model_names( - sql=sql, file_label=file_label, scenario=True, syntax=syntax - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/sharing/binding.py b/src/sqlbuild/compiler/compile/_helpers/sharing/binding.py deleted file mode 100644 index d337b2e6fd..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/sharing/binding.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Exact sharing of binding-mode native analyses between models with equal inputs.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile._helpers.analysis.reference_names import ( - _analysis_reference_name, - _lineage_resource_type, -) -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, -) -from sqlbuild.compiler.compile.types import CompiledResourceType - - -def lineage_reference_map( - references: tuple[CompileSqlReference, ...], -) -> dict[str, tuple[CompiledResourceType, str]]: - reference_map: dict[str, tuple[CompiledResourceType, str]] = {} - reference: CompileSqlReference - for reference in references: - resource_type: CompiledResourceType | None = _lineage_resource_type(reference) - if resource_type is None: - continue - reference_map[_analysis_reference_name(reference)] = ( - resource_type, - reference.ref_name, - ) - return reference_map diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py index f9e2f06b19..47743fb45b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py @@ -5,18 +5,10 @@ from functools import lru_cache import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references -from sqlbuild.compiler.compile.constants import ( - DEFAULT_SQL_TEST_MODE, - OMITTED_CEREMONIAL_SELECT_SQL, -) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileSqlTestCte, - SqlReferenceOrigin, ) -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -32,61 +24,3 @@ def extract_unclassified_sql_test_ctes( if error is not None: raise CompileInputError(error) return tuple(CompileSqlTestCte(name=name, sql_body=body) for name, body in ctes) - - -def extract_sql_test_expected_model_names( - *, - sql: str, - file_label: str, - syntax: SqlLexicalSyntax, - mode: SqlTestMode = DEFAULT_SQL_TEST_MODE, -) -> tuple[str, ...]: - """Extract explicit expected-model relationships without inspecting CTE bodies.""" - - if mode is not SqlTestMode.MODEL: - return () - return scanned_expected_model_names( - sql=sql, file_label=file_label, scenario=False, syntax=syntax - ) - - -def scanned_expected_model_names( - *, sql: str, file_label: str, scenario: bool, syntax: SqlLexicalSyntax -) -> tuple[str, ...]: - """A SQL test's or scenario's `__expected__` names, raising the scan's error.""" - - error: str | None - names: list[str] - error, names = _native.scope_expected_model_names( - [(sql, file_label)], scenario, syntax.native_mapping - )[0] - if error is not None: - raise CompileInputError(error) - return tuple(names) - - -def extract_assertion_target_model_names( - *, - assertion_sql: tuple[str, ...], - syntax: SqlLexicalSyntax, - origin: SqlReferenceOrigin | None = None, -) -> tuple[str, ...]: - """Extract assertion model targets in authored order using canonical references.""" - - targets: list[str] = [] - for sql in assertion_sql: - targets.extend( - reference.ref_name - for reference in extract_sql_references(sql=sql, syntax=syntax, origin=origin) - if reference.ref_kind == SqlReferenceKind.REF - ) - return tuple(dict.fromkeys(targets)) - - -def complete_omitted_ceremonial_select(*, sql: str, syntax: SqlLexicalSyntax) -> str: - """Return test or scenario SQL as one statement, adding an omitted trailing `SELECT 1`.""" - - offset: int | None = _native.omitted_ceremonial_select(sql, syntax.native_mapping) - if offset is None: - return sql - return f"{sql[:offset]}{OMITTED_CEREMONIAL_SELECT_SQL}{sql[offset:]}" diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py index c7d03e1845..8cb95bab87 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/extraction_errors.py @@ -6,27 +6,16 @@ from dataclasses import dataclass from pathlib import Path -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.refs.references import ( - reference_call_location, - reference_call_syntax_key, - scan_sql_reference_calls, -) from sqlbuild.compiler.compile._helpers.sql_tests.native import ( authored_sql_test_cte_batch, - authored_sql_test_ctes, ) -from sqlbuild.compiler.compile.constants import REFERENCE_CALL_SYNTAX_CODE from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestExtractionError -from sqlbuild.compiler.compile.models import CompilerDiagnostic, SqlReferenceScan -from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity from sqlbuild.compiler.discovery.models import ( DiscoveredSqlScenarioFile, DiscoveredSqlTestBlock, DiscoveredSqlTestFile, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceLocation _TAB_SIZE: int = 8 _LINE_BREAK: str = "\n" @@ -136,57 +125,6 @@ def _located_error( ) -def report_authored_invalid_calls( - *, - test_file: DiscoveredSqlTestFile, - test_block: DiscoveredSqlTestBlock, - actual_cte_name: str, - syntax: SqlLexicalSyntax, -) -> int: - """Report each malformed reference call outside the actual CTE at its own location.""" - - ctes: tuple[tuple[str, int, str], ...] = authored_sql_test_ctes( - sql=test_block.sql_body, file_label=str(test_file.relative_path), syntax=syntax - ) - reported: int = 0 - name: str - body_start: int - body: str - for name, body_start, body in ctes: - if name == actual_cte_name: - continue - scan: SqlReferenceScan = scan_sql_reference_calls(sql=body, syntax=syntax) - for invalid_call in scan.invalid_calls: - offset: int | None = authored_file_offset( - test_file=test_file, - test_block=test_block, - body_offset=body_start + invalid_call.start, - ) - if offset is None: - return reported - location: SourceLocation = reference_call_location( - path=test_file.relative_path, - text=test_file.contents, - start=offset, - call=invalid_call.call, - ) - report_compile_diagnostic( - key=reference_call_syntax_key( - file_path=test_file.file_path, call=invalid_call.call, location=location - ), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=REFERENCE_CALL_SYNTAX_CODE, - message=invalid_call.message, - location=location, - help=invalid_call.help, - ), - ) - reported += 1 - return reported - - def authored_file_offset( *, test_file: DiscoveredSqlTestFile, test_block: DiscoveredSqlTestBlock, body_offset: int ) -> int | None: diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py deleted file mode 100644 index fa806f2e8b..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py +++ /dev/null @@ -1,423 +0,0 @@ -"""Relation references in the SQL-test helper CTEs a test reads, and what cannot resolve.""" - -from __future__ import annotations - -import re -from collections.abc import Iterable - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.refs.references import ( - reference_call_location, - scan_sql_reference_calls, -) -from sqlbuild.compiler.compile.constants import ( - DBT_REF_TEST_CTE_PREFIX, - REF_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, - SQL_TEST_HELPER_REFERENCE_CODE, - TABLE_FN_TEST_CTE_PREFIX, -) -from sqlbuild.compiler.compile.models import ( - CompileModelSqlTestCtes, - CompilerDiagnostic, - CompileSqlReference, - CompileSqlTestCte, - SqlTestCteGraph, - SqlTestReads, -) -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceLocation - -_MOCK_CTE_PREFIXES: tuple[str, ...] = ( - REF_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - DBT_REF_TEST_CTE_PREFIX, - TABLE_FN_TEST_CTE_PREFIX, -) -_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"[A-Za-z_][\w$]*") - - -def sql_test_cte_graph( - *, - authored_ctes: tuple[CompileSqlTestCte, ...], - reader_ctes: tuple[CompileSqlTestCte, ...], -) -> SqlTestCteGraph: - """Return the authored CTEs each test CTE reads, ignoring names its own nested CTEs define.""" - - ctes: dict[str, CompileSqlTestCte] = {cte.name.casefold(): cte for cte in authored_ctes} - keys: frozenset[str] = frozenset(ctes) - reads: dict[str, tuple[str, ...]] = {} - parsed_reads: dict[str, tuple[str, ...]] = {} - for key, cte in ctes.items(): - parsed: tuple[str, ...] | None = _parsed_reads(sql=cte.sql_body, keys=keys - {key}) - parsed_reads[key] = parsed or () - reads[key] = ( - parsed if parsed is not None else _token_reads(sql=cte.sql_body, keys=keys - {key}) - ) - reader_reads: list[str] = [] - for reader in reader_ctes: - parsed_reader: tuple[str, ...] | None = _parsed_reads(sql=reader.sql_body, keys=keys) - reader_reads.extend( - parsed_reader - if parsed_reader is not None - else _token_reads(sql=reader.sql_body, keys=keys) - ) - return SqlTestCteGraph( - ctes=ctes, - reads=reads, - parsed_reads=parsed_reads, - reader_reads=tuple(dict.fromkeys(reader_reads)), - ) - - -def reachable_cte_keys(*, graph: SqlTestCteGraph, roots: Iterable[str]) -> tuple[str, ...]: - """Return the authored CTEs reachable from ``roots``, including the roots, in visit order.""" - - pending: list[str] = [root for root in roots if root in graph.ctes] - visited: list[str] = list(dict.fromkeys(pending)) - while pending: - key: str = pending.pop(0) - for dependency in graph.reads.get(key, ()): - if dependency not in visited: - visited.append(dependency) - pending.append(dependency) - return tuple(visited) - - -def sql_test_reads(*, payload: CompileModelSqlTestCtes, syntax: SqlLexicalSyntax) -> SqlTestReads: - """Return the helpers a test reads and the unmocked models they and expected rows read.""" - - graph: SqlTestCteGraph = sql_test_cte_graph( - authored_ctes=payload.authored_ctes, reader_ctes=_reader_ctes(payload) - ) - read_helpers: tuple[str, ...] = _read_helper_keys(graph) - mocked: frozenset[str] = frozenset(payload.mock_model_names) - sqls: tuple[str, ...] = ( - *(cte.sql_body for cte in payload.expected_ctes), - *(graph.ctes[key].sql_body for key in read_helpers), - ) - targets: list[str] = [] - for sql in sqls: - targets.extend( - reference.ref_name - for reference in _references(sql=sql, syntax=syntax) - if reference.ref_kind == SqlReferenceKind.REF and reference.ref_name not in mocked - ) - return SqlTestReads( - read_helper_names=tuple(graph.ctes[key].name for key in read_helpers), - reference_target_model_names=tuple(dict.fromkeys(targets)), - ) - - -def report_unresolvable_test_references( - *, - payload: CompileModelSqlTestCtes, - test_file: DiscoveredSqlTestFile, - test_block: DiscoveredSqlTestBlock, - known_model_names: set[str], - syntax: SqlLexicalSyntax, -) -> bool: - """Report unresolvable references in expected rows, assertions and read helpers, or cycles.""" - - graph: SqlTestCteGraph = sql_test_cte_graph( - authored_ctes=payload.authored_ctes, reader_ctes=_reader_ctes(payload) - ) - read_helpers: tuple[str, ...] = _read_helper_keys(graph) - readers: tuple[tuple[str, CompileSqlTestCte], ...] = ( - *(("helper CTE", graph.ctes[key]) for key in read_helpers), - *(("expected CTE", cte) for cte in payload.expected_ctes), - *(("assertion CTE", cte) for cte in payload.assertion_ctes), - ) - reporter: _HelperDiagnostics = _HelperDiagnostics(test_file=test_file, test_block=test_block) - mocked: dict[SqlReferenceKind, frozenset[str]] = { - SqlReferenceKind.REF: frozenset(payload.mock_model_names), - SqlReferenceKind.SOURCE: frozenset(payload.mock_source_names), - SqlReferenceKind.SEED: frozenset(payload.mock_seed_names), - SqlReferenceKind.DBT_REF: frozenset(payload.mock_dbt_ref_names), - } - for label, cte in readers: - for reference in _references(sql=cte.sql_body, syntax=syntax): - kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) - if kind not in mocked: - continue - mock_name: str = ( - f"{reference.ref_package}__{reference.ref_name}" - if reference.ref_package is not None - else reference.ref_name - ) - if mock_name in mocked[kind]: - continue - if kind is SqlReferenceKind.REF: - if reference.ref_name in known_model_names: - continue - reporter.report( - cte_name=cte.name, - call=kind.example_call(reference.ref_name, quote='"'), - message=( - f"SQL test {label} '{cte.name}' references unknown model " - f"'{reference.ref_name}'" - ), - help=( - 'Read a model defined in the project, for example __ref(""), ' - f"or remove the reference from {label} '{cte.name}'." - ), - ) - continue - call: str = _reference_call(reference) - reporter.report( - cte_name=cte.name, - call=call, - message=( - f"SQL test {label} '{cte.name}' calls {call}, which the test does not " - "mock, so the test query cannot resolve it" - ), - help=( - f"Mock it in the test, for example {kind.fixture_cte_prefix}{mock_name} " - f"AS (SELECT ...), or remove the call from {label} '{cte.name}'." - ), - ) - cycle: tuple[str, ...] | None = ( - _first_cycle(graph=graph, keys=reachable_cte_keys(graph=graph, roots=graph.reader_reads)) - if read_helpers - else None - ) - if cycle is not None: - names: tuple[str, ...] = tuple(graph.ctes[key].name for key in cycle) - path: str = " -> ".join(f"'{name}'" for name in names) - reporter.report( - cte_name=names[0], - call=None, - message=( - f"SQL test CTE '{names[0]}' reads itself through {path}, so the test query " - "cannot define its CTEs in dependency order" - ), - help=( - "Break the cycle by moving the shared rows into a helper CTE that reads neither, " - "for example shared_rows AS (SELECT ...), and read shared_rows from both CTEs." - ), - ) - return reporter.reported - - -def report_test_without_target_model( - *, - payload: CompileModelSqlTestCtes, - test_file: DiscoveredSqlTestFile, - test_block: DiscoveredSqlTestBlock, - syntax: SqlLexicalSyntax, -) -> None: - """Report a model test whose checks read only mocks, so it would run no model.""" - - mocked: dict[SqlReferenceKind, frozenset[str]] = { - SqlReferenceKind.REF: frozenset(payload.mock_model_names), - SqlReferenceKind.SOURCE: frozenset(payload.mock_source_names), - SqlReferenceKind.SEED: frozenset(payload.mock_seed_names), - SqlReferenceKind.DBT_REF: frozenset(payload.mock_dbt_ref_names), - SqlReferenceKind.TABLE_FUNCTION: frozenset(payload.mock_table_function_names), - } - graph: SqlTestCteGraph = sql_test_cte_graph( - authored_ctes=payload.authored_ctes, reader_ctes=_reader_ctes(payload) - ) - read_mocks: list[str] = [ - graph.ctes[key].name - for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads) - if graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES) - ] - first_calls: dict[SqlReferenceKind, tuple[str, str]] = {} - called_models: set[str] = set() - for cte in _reader_ctes(payload): - for reference in _references(sql=cte.sql_body, syntax=syntax): - kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) - if reference.ref_name not in mocked.get(kind, frozenset()): - continue - read_mocks.append(f"{kind.fixture_cte_prefix}{reference.ref_name}") - if kind is SqlReferenceKind.REF: - called_models.add(reference.ref_name) - if kind is not SqlReferenceKind.TABLE_FUNCTION: - first_calls.setdefault(kind, (cte.name, _reference_call(reference))) - mocks: tuple[str, ...] = tuple(dict.fromkeys(read_mocks)) - mocked_models: tuple[str, ...] = tuple( - mock.removeprefix(REF_TEST_CTE_PREFIX) - for mock in mocks - if mock.startswith(REF_TEST_CTE_PREFIX) - ) - reader: CompileSqlTestCte | None = next(iter(_reader_ctes(payload)), None) - located: tuple[str, str] | None = first_calls.get(SqlReferenceKind.REF) or ( - _mock_name_read(readers=_reader_ctes(payload), mocks=mocks) - or next(iter(first_calls.values()), None) - ) - test_name: str = test_block.name or test_file.relative_path.stem - message: str - help_text: str - if mocked_models: - plural: str = "s" if len(mocked_models) > 1 else "" - owner: str = "models'" if plural else "model's" - verb: str = "keep" if called_models.issuperset(mocked_models) else "call" - mocked_ctes: str = ", ".join(f"{REF_TEST_CTE_PREFIX}{name}" for name in mocked_models) - calls: str = " and ".join( - SqlReferenceKind.REF.example_call(name, quote='"') for name in mocked_models - ) - message = ( - f"SQL test '{test_name}' mocks the model{plural} it tests ({mocked_ctes}), so the " - "test has no model to run against" - ) - help_text = ( - f"Mock the {owner} inputs instead (for example __source__ or " - f"__ref__) and {verb} {calls} in the __assert__ or __expected__ CTE, " - "or remove the test." - ) - else: - reads: str = f"reads only mocks ({', '.join(mocks)})" if mocks else "reads no model" - message = f"SQL test '{test_name}' {reads}, so the test has no model to run against" - help_text = ( - 'Call __ref("") in an __assert__ or __expected__ CTE so the test ' - "runs that model with its mocks, or remove the test." - ) - _HelperDiagnostics(test_file=test_file, test_block=test_block).report( - cte_name=located[0] if located else (reader.name if reader is not None else test_name), - call=located[1] if located else None, - message=message, - help=help_text, - ) - - -def _mock_name_read( - *, readers: tuple[CompileSqlTestCte, ...], mocks: tuple[str, ...] -) -> tuple[str, str] | None: - """Return the first check reading a mocked model by CTE name, with that name.""" - - for mock in mocks: - if not mock.startswith(REF_TEST_CTE_PREFIX): - continue - for reader in readers: - if re.search(rf"(? tuple[CompileSqlTestCte, ...]: - return (*payload.expected_ctes, *payload.assertion_ctes) - - -def _read_helper_keys(graph: SqlTestCteGraph) -> tuple[str, ...]: - return tuple( - key - for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads) - if not graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES) - ) - - -def _reference_call(reference: CompileSqlReference) -> str: - kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) - if kind is SqlReferenceKind.DBT_REF and reference.ref_package is not None: - return kind.example_call(reference.ref_package, reference.ref_name, quote='"') - return kind.example_call(reference.ref_name, quote='"') - - -def _references(*, sql: str, syntax: SqlLexicalSyntax) -> tuple[CompileSqlReference, ...]: - return scan_sql_reference_calls(sql=sql, syntax=syntax).references - - -def _parsed_reads(*, sql: str, keys: frozenset[str]) -> tuple[str, ...] | None: - """Return unqualified table reads of ``keys`` not shadowed by a nested CTE, or None.""" - - reads: list[str] | None = _native.sql_test_helper_reads(sql, sorted(keys)) - return None if reads is None else tuple(reads) - - -def _token_reads(*, sql: str, keys: frozenset[str]) -> tuple[str, ...]: - tokens: tuple[str, ...] = tuple( - dict.fromkeys(match.casefold() for match in _IDENTIFIER_PATTERN.findall(sql)) - ) - return tuple(token for token in tokens if token in keys) - - -def _first_cycle(*, graph: SqlTestCteGraph, keys: tuple[str, ...]) -> tuple[str, ...] | None: - """Return the first parsed read cycle among ``keys``, closed on its first CTE.""" - - allowed: frozenset[str] = frozenset(keys) - for origin in keys: - pending: list[tuple[str, ...]] = [(origin,)] - visited: set[str] = set() - while pending: - path: tuple[str, ...] = pending.pop(0) - for dependency in graph.parsed_reads.get(path[-1], ()): - if dependency == origin: - return (*path, origin) - if dependency in allowed and dependency not in visited: - visited.add(dependency) - pending.append((*path, dependency)) - return None - - -def sql_test_cte_location( - *, - test_file: DiscoveredSqlTestFile, - test_block: DiscoveredSqlTestBlock, - cte_name: str, - call: str | None, -) -> SourceLocation: - """Locate a call inside one CTE of a SQL test block, else the CTE header, else the block.""" - - contents: str = test_file.contents - block_offset: int = max(contents.find(test_block.sql_body), 0) - header: re.Match[str] | None = re.compile( - rf"(? None: - self._test_file: DiscoveredSqlTestFile = test_file - self._test_block: DiscoveredSqlTestBlock = test_block - self._test_name: str = test_block.name or test_file.relative_path.stem - self.reported: bool = False - - def report(self, *, cte_name: str, call: str | None, message: str, help: str) -> None: - location: SourceLocation = sql_test_cte_location( - test_file=self._test_file, test_block=self._test_block, cte_name=cte_name, call=call - ) - self.reported = True - report_compile_diagnostic( - key=( - SQL_TEST_HELPER_REFERENCE_CODE, - self._test_file.relative_path.as_posix(), - f"{location.line:09d}:{location.column:09d}", - message, - ), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=SQL_TEST_HELPER_REFERENCE_CODE, - message=message, - resource_type=CompiledResourceType.SQL_TEST, - resource_name=self._test_name, - location=location, - help=help, - ), - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py index 0ed006fb53..1ab0b45a23 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py @@ -2,26 +2,15 @@ from __future__ import annotations -from collections.abc import Callable, Sequence from typing import Protocol, cast import orjson import sqlbuild._native as _native -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.constants import SQL_TEST_CTE_SCAN_ALGORITHM from sqlbuild.compiler.compile.exceptions import ( CompileInputError, - NativeSqlTestResponseError, SqlTestExtractionError, ) -from sqlbuild.compiler.compile.models import ( - CompileDirectLogicSqlTestCtes, - CompileModelSqlTestCtes, - CompileSqlTestCte, - CompileSqlTestCtes, -) -from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -29,7 +18,6 @@ class _NativeSqlTestModule(Protocol): def extract_sql_tests_json(self, request_json: str) -> str: ... -_DIRECT_KIND: str = "direct" _SYNTAX_FIELDS: dict[str, str] = { "backslash_escape_quotes": "backslashEscapeQuotes", "escape_string_prefix": "escapeStringPrefix", @@ -38,129 +26,9 @@ def extract_sql_tests_json(self, request_json: str) -> str: ... "nested_block_comments": "nestedBlockComments", "line_comment_prefixes": "lineCommentPrefixes", } -_MODEL_KIND: str = "model" -_PAIR_LENGTH: int = 2 _AUTHORED_CTE_LENGTH: int = 3 -def extract_expanded_sql_tests( - *, - tests: tuple[tuple[str, str, SqlTestMode], ...], - syntax: SqlLexicalSyntax, - scan_cache: SqlTestScanCache | None = None, -) -> tuple[CompileSqlTestCtes, ...]: - """Extract expanded tests natively, reusing whole stored results of unchanged files.""" - - store: SqlTestScanCache = scan_cache or SqlTestScanCache(cache_dir=None) - indexes_by_file: dict[str, list[int]] = {} - for index, (_sql, file_label, _mode) in enumerate(tests): - indexes_by_file.setdefault(file_label, []).append(index) - results: list[CompileSqlTestCtes | None] = [None] * len(tests) - parts_by_file: dict[str, list[str]] = {} - for file_label, indexes in indexes_by_file.items(): - parts: list[str] = [file_label] - for index in indexes: - parts.extend((tests[index][0], tests[index][2].value)) - modes: tuple[SqlTestMode, ...] = tuple(tests[index][2] for index in indexes) - stored: tuple[CompileSqlTestCtes, ...] | None = store.read( - algorithm=SQL_TEST_CTE_SCAN_ALGORITHM, - syntax=syntax, - parts=parts, - decode=_stored_file_tests(modes=modes), - ) - if stored is None: - parts_by_file[file_label] = parts - continue - for index, test_ctes in zip(indexes, stored, strict=True): - results[index] = test_ctes - missing: list[int] = [] - for file_label in parts_by_file: - missing.extend(indexes_by_file[file_label]) - missing.sort() - if missing: - try: - items: list[object] = _native_batch( - tests=[ - { - "sql": tests[index][0], - "fileLabel": tests[index][1], - "mode": tests[index][2].value, - "raw": False, - } - for index in missing - ], - syntax=syntax, - ) - except SqlTestExtractionError as error: - error.test_index = missing[error.test_index] - raise - items_by_index: dict[int, object] = dict(zip(missing, items, strict=True)) - for file_label, parts in parts_by_file.items(): - file_items: list[object] = [ - items_by_index[index] for index in indexes_by_file[file_label] - ] - for index, item in zip(indexes_by_file[file_label], file_items, strict=True): - results[index] = _test_ctes_from_native(item) - store.write( - algorithm=SQL_TEST_CTE_SCAN_ALGORITHM, - syntax=syntax, - parts=parts, - value=orjson.dumps(file_items), - ) - return tuple(result for result in results if result is not None) - - -def _stored_file_tests( - *, modes: tuple[SqlTestMode, ...] -) -> Callable[[bytes], tuple[CompileSqlTestCtes, ...] | None]: - """Decode one file's stored tests, rejecting anything that is not their exact shape.""" - - def decode(value: bytes) -> tuple[CompileSqlTestCtes, ...] | None: - try: - items: object = orjson.loads(value) - if not isinstance(items, list) or len(cast(list[object], items)) != len(modes): - return None - decoded: tuple[CompileSqlTestCtes, ...] = tuple( - _test_ctes_from_native(item) for item in cast(list[object], items) - ) - except (orjson.JSONDecodeError, CompileInputError): - return None - if any(test_ctes.mode is not mode for test_ctes, mode in zip(decoded, modes, strict=True)): - return None - return decoded - - return decode - - -def native_sql_test_ctes( - *, texts: Sequence[tuple[str, str]], syntax: SqlLexicalSyntax -) -> list[tuple[tuple[str, str], ...] | str]: - """Return each `(sql, file label)` test body's CTEs or the scanner's error.""" - - if not texts: - return [] - outcomes: list[tuple[str | None, list[tuple[str, str]]]] = _native.scope_test_ctes( - list(texts), syntax.native_mapping - ) - return [outcome[0] if outcome[0] is not None else tuple(outcome[1]) for outcome in outcomes] - - -def extract_unexpanded_sql_test( - *, sql: str, file_label: str, mode: SqlTestMode, syntax: SqlLexicalSyntax -) -> tuple[CompileSqlTestCtes, bool]: - """Extract a direct-logic test before expansion; also report whether it has P012 calls.""" - - return _extract_natively(tests=((sql, file_label, mode, True),), syntax=syntax)[0] - - -def authored_sql_test_ctes( - *, sql: str, file_label: str, syntax: SqlLexicalSyntax -) -> tuple[tuple[str, int, str], ...]: - """Read an authored block's CTEs with body offsets; only a CTE-name error is raised.""" - - return authored_sql_test_cte_batch(tests=((sql, file_label),), syntax=syntax)[0] - - def authored_sql_test_cte_batch( *, tests: tuple[tuple[str, str], ...], syntax: SqlLexicalSyntax, scenarios: bool = False ) -> tuple[tuple[tuple[str, int, str], ...], ...]: @@ -203,23 +71,6 @@ def _authored_cte(value: object) -> tuple[str, int, str]: return value[0], value[1], value[2] -def _extract_natively( - *, tests: tuple[tuple[str, str, SqlTestMode, bool], ...], syntax: SqlLexicalSyntax -) -> tuple[tuple[CompileSqlTestCtes, bool], ...]: - results: list[object] = _native_batch( - tests=[ - {"sql": sql, "fileLabel": file_label, "mode": mode.value, "raw": raw} - for sql, file_label, mode, raw in tests - ], - syntax=syntax, - ) - return tuple((_test_ctes_from_native(item), _invalid_calls(item)) for item in results) - - -def _invalid_calls(item: object) -> bool: - return isinstance(item, dict) and cast(dict[str, object], item).get("invalidCalls") is True - - def _native_batch(*, tests: list[dict[str, object]], syntax: SqlLexicalSyntax) -> list[object]: native_syntax: dict[str, object] = syntax.native_mapping request_json: str = orjson.dumps( @@ -255,90 +106,3 @@ def _extraction_error(error: dict[str, object]) -> SqlTestExtractionError: token=token if isinstance(token, str) else None, token_offset=token_offset if isinstance(token_offset, int) else None, ) - - -def _test_ctes_from_native(value: object) -> CompileSqlTestCtes: - if not isinstance(value, dict): - raise CompileInputError("native SQL test extraction returned an invalid test response") - payload: dict[str, object] = cast(dict[str, object], value) - try: - mode: SqlTestMode = SqlTestMode(_required_string(value=payload, key="mode")) - kind: str = _required_string(value=payload, key="kind") - if kind == _DIRECT_KIND: - return CompileSqlTestCtes( - mode=mode, - payload=CompileDirectLogicSqlTestCtes( - mode=mode, - helper_ctes=_ctes(payload.get("helpers")), - actual_cte=_cte(payload.get("actual")), - expected_cte=_cte(payload.get("expected")), - ), - ) - if kind != _MODEL_KIND or mode is not SqlTestMode.MODEL: - raise NativeSqlTestResponseError("native SQL test extraction returned an invalid kind") - return CompileSqlTestCtes( - mode=mode, - payload=CompileModelSqlTestCtes( - authored_ctes=_ctes(payload.get("authored")), - macro_mocks=dict(_string_pairs(payload.get("macroMocks"))), - mock_model_names=_strings(payload.get("mockModels")), - mock_source_names=_strings(payload.get("mockSources")), - mock_seed_names=_strings(payload.get("mockSeeds")), - mock_dbt_ref_names=_strings(payload.get("mockDbtRefs")), - mock_table_function_names=_strings(payload.get("mockTableFunctions")), - expected_ctes=_ctes(payload.get("expected")), - expected_model_names=_strings(payload.get("expectedModels")), - assertion_ctes=_ctes(payload.get("assertions")), - assertion_names=_strings(payload.get("assertionNames")), - ), - ) - except (KeyError, TypeError, ValueError, NativeSqlTestResponseError): - raise CompileInputError( - "native SQL test extraction returned an invalid test response" - ) from None - - -def _required_string(*, value: dict[str, object], key: str) -> str: - item: object = value[key] - if not isinstance(item, str): - raise NativeSqlTestResponseError("native SQL test extraction expected a string") - return item - - -def _strings(value: object) -> tuple[str, ...]: - if not isinstance(value, list) or not all(isinstance(item, str) for item in value): - raise NativeSqlTestResponseError("native SQL test extraction expected a string list") - return tuple(cast(list[str], value)) - - -def _cte(value: object) -> CompileSqlTestCte: - if ( - not isinstance(value, list) - or len(value) != _PAIR_LENGTH - or not isinstance(value[0], str) - or not isinstance(value[1], str) - ): - raise NativeSqlTestResponseError("native SQL test extraction expected one CTE pair") - return CompileSqlTestCte(name=value[0], sql_body=value[1]) - - -def _ctes(value: object) -> tuple[CompileSqlTestCte, ...]: - if not isinstance(value, list): - raise NativeSqlTestResponseError("native SQL test extraction expected a CTE list") - return tuple(_cte(item) for item in value) - - -def _string_pairs(value: object) -> tuple[tuple[str, str], ...]: - if not isinstance(value, list): - raise NativeSqlTestResponseError("native SQL test extraction expected a pair list") - pairs: list[tuple[str, str]] = [] - for item in value: - if ( - not isinstance(item, list) - or len(item) != _PAIR_LENGTH - or not isinstance(item[0], str) - or not isinstance(item[1], str) - ): - raise NativeSqlTestResponseError("native SQL test extraction expected string pairs") - pairs.append((item[0], item[1])) - return tuple(pairs) diff --git a/src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py b/src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py deleted file mode 100644 index e59c882b93..0000000000 --- a/src/sqlbuild/compiler/compile/classes/macro_expansion_facts.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Facts recorded while expanding the macros of one authored SQL string.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.macro_bridge.constants import ARGUMENT_REFERENCE_EVENT -from sqlbuild.compiler.macro_bridge.types import MacroCallEvent -from sqlbuild.compiler.scopes.models import DeclarationIdentity, UsageRecord -from sqlbuild.python_nodes.models import SqlResourceRef - - -class MacroExpansionFacts: - """Facts one authored string's macro expansion records, in encounter order.""" - - def __init__(self) -> None: - self.dependencies: list[DeclarationIdentity] = [] - self.usages: list[UsageRecord] = [] - self.relations: dict[SqlResourceRef, int] = {} - self.argument_references: dict[SqlResourceRef, None] = {} - self.call_site_refs: list[set[SqlResourceRef]] = [] - self.rendering_approved: list[frozenset[SqlResourceRef]] = [] - self.event_log: list[MacroCallEvent] | None = None - self.memo_blocked: bool = False - - def begin_event_log(self) -> MacroExpansionFacts: - """Start recording the consumer-independent facts of one top-level macro call.""" - - self.event_log = [] - self.memo_blocked = False - return self - - def end_event_log(self) -> tuple[tuple[MacroCallEvent, ...], bool]: - """Stop recording and return the facts and whether the call may be replayed.""" - - events: tuple[MacroCallEvent, ...] = tuple(self.event_log or ()) - self.event_log = None - return events, not self.memo_blocked - - def note_event(self, event: MacroCallEvent) -> None: - """Record one fact of the macro call being executed, when recording.""" - - if self.event_log is not None: - self.event_log.append(event) - - def block_memo(self) -> None: - """Mark the executing call as dependent on its consumer, so it is never replayed.""" - - self.memo_blocked = True - - def add_dependency(self, identity: DeclarationIdentity) -> MacroExpansionFacts: - """Record a resolved dependency in encounter order.""" - - self.dependencies.append(identity) - return self - - def add_usage(self, usage: UsageRecord) -> MacroExpansionFacts: - """Record a resolved macro-to-macro edge in encounter order.""" - - self.usages.append(usage) - return self - - def record_call_site_refs(self, refs: tuple[SqlResourceRef, ...]) -> None: - """Record references written as arguments of every macro call being expanded.""" - - for ref in refs: - self.note_event((ARGUMENT_REFERENCE_EVENT, ref.kind.value, ref.name)) - self.argument_references.update(dict.fromkeys(refs)) - for approved in self.call_site_refs: - approved.update(refs) - - def open_call_site(self) -> MacroExpansionFacts: - """Start collecting the references written as one macro call's arguments.""" - - self.call_site_refs.append(set()) - return self - - def close_call_site(self) -> frozenset[SqlResourceRef]: - """Stop collecting for the innermost macro call and return its argument references.""" - - return frozenset(self.call_site_refs.pop()) - - def begin_rendering(self, approved: frozenset[SqlResourceRef]) -> MacroExpansionFacts: - """Approve the call-site argument references of the macro about to run.""" - - self.rendering_approved.append(approved) - return self - - def end_rendering(self) -> MacroExpansionFacts: - """Restore the approvals of the enclosing macro after one macro returns.""" - - self.rendering_approved.pop() - return self - - def render_relation(self, ref: object) -> str: - """Render a reference a macro formats; only call-site arguments become placeholders.""" - - from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( - reference_call_text, - relation_placeholder_text, - ) - - if not isinstance(ref, SqlResourceRef): - raise CompileInputError("Only typed resource references render as relations") - if not self.rendering_approved or ref not in self.rendering_approved[-1]: - return reference_call_text(ref) - return relation_placeholder_text(self.relations.setdefault(ref, len(self.relations))) - - def render_relation_placeholders(self, sql: str) -> str: - """Replace typed reference placeholders with the reference call written at the call site.""" - - from sqlbuild.compiler.compile._helpers.explicit_references.macro_arguments import ( - render_relation_placeholders, - ) - - return render_relation_placeholders(sql=sql, relations=self.relations) diff --git a/src/sqlbuild/compiler/compile/classes/python_model_analysis.py b/src/sqlbuild/compiler/compile/classes/python_model_analysis.py index b3005470bc..3b5315e234 100644 --- a/src/sqlbuild/compiler/compile/classes/python_model_analysis.py +++ b/src/sqlbuild/compiler/compile/classes/python_model_analysis.py @@ -4,30 +4,21 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile._helpers.analysis.cache import record_analysis_cache_metrics -from sqlbuild.compiler.compile._helpers.analysis.reference_names import ( - substitute_placeholder_defaults, - table_function_analysis_name, -) -from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import names_set_operation from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( - binding_relation_names, binding_schema_for_model, ) from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( cursor_intrinsics_analysis_sql, ) -from sqlbuild.compiler.compile._helpers.sharing.binding import lineage_reference_map from sqlbuild.compiler.compile.models import ( - CompileModelInput, - CompileSqlReference, ModelSqlAnalysisRequest, ) +from sqlbuild.compiler.compile.types import ModelInput from sqlbuild.compiler.planner.constants import ( SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, SNAPSHOT_DEFAULT_VALID_TO_COLUMN, ) from sqlbuild.compiler.planner.types import ContractPolicy, MaterializationType -from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -37,7 +28,7 @@ class PythonModelAnalysis: def __init__( self, *, - model_inputs: tuple[CompileModelInput, ...], + model_inputs: tuple[ModelInput, ...], inference_profile: ExpressionInferenceProfile, complete_binding_schemas: dict[str, dict[str, str]], ) -> None: @@ -90,24 +81,15 @@ def record_uncached(self) -> None: def _model_row(request: ModelSqlAnalysisRequest) -> tuple[object, ...]: - model_input: CompileModelInput = request.model_input - references: tuple[CompileSqlReference, ...] = model_input.references + model_input: ModelInput = request.model_input + rows: list[tuple[str, str, str | None, int | None]] = model_input.reference_rows values: dict[str, object] = model_input.config.values return ( model_input.model_file.file_path.stem, request.query_sql, list((request.placeholders or {}).items()), - [ - (_analysis_name(reference), reference.ref_kind == SqlReferenceKind.REF) - for reference in references - ], - [ - (name, resource_type.value, resource_name) - for name, (resource_type, resource_name) in lineage_reference_map(references).items() - ], - sorted(binding_relation_names(references)), + [(str(kind), name) for kind, name, _, _ in rows], _recovers_cte_facts(model_input), - names_set_operation(request.query_sql), ( ( str(values.get("valid_from_column") or SNAPSHOT_DEFAULT_VALID_FROM_COLUMN), @@ -116,13 +98,6 @@ def _model_row(request: ModelSqlAnalysisRequest) -> tuple[object, ...]: if values.get("materialized") == MaterializationType.SNAPSHOT else None ), - ( - substitute_placeholder_defaults( - query_sql=request.query_sql, placeholders=request.placeholders - ) - if request.placeholders - else request.query_sql - ), [ PythonModelAnalysis.family_row(family) for family in ( @@ -134,13 +109,7 @@ def _model_row(request: ModelSqlAnalysisRequest) -> tuple[object, ...]: ) -def _analysis_name(reference: CompileSqlReference) -> str: - if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION: - return table_function_analysis_name(reference.ref_name) - return reference.ref_name - - -def _placeholders(model_input: CompileModelInput) -> dict[str, str] | None: +def _placeholders(model_input: ModelInput) -> dict[str, str] | None: raw_placeholders: object | None = model_input.config.values.get("placeholders") return ( {str(key): str(value) for key, value in raw_placeholders.items()} @@ -149,7 +118,7 @@ def _placeholders(model_input: CompileModelInput) -> dict[str, str] | None: ) -def _recovers_cte_facts(model_input: CompileModelInput) -> bool: +def _recovers_cte_facts(model_input: ModelInput) -> bool: return model_input.config.values.get("contract") == ContractPolicy.ENFORCED or ( model_input.schema_entry is not None and bool(model_input.schema_entry.type_enforcement) ) diff --git a/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py b/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py index 3d0c265b2a..6ab8504125 100644 --- a/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py +++ b/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py @@ -9,9 +9,8 @@ get_complete_schema_binding_request, ) from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( - _audit_errors, - _names, - _sql_test_columns, + names, + sql_test_columns, ) from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( known_declared_types, @@ -19,7 +18,8 @@ ) from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes from sqlbuild.compiler.compile._helpers.diagnostics.resource_sql import get_resource_sql_diagnostics -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case from sqlbuild.compiler.sql_analysis.models import SqlSchemaValidationRequest @@ -28,14 +28,14 @@ class SemanticCompletionInputs: """Expose the shapes, revalidation requests and metadata checks semantic completion reads.""" @staticmethod - def shapes(project: CompiledProject) -> dict[str, dict[str, str]]: + def shapes(project: CompiledProjectView) -> dict[str, dict[str, str]]: """Closed relation shapes, as type recovery, recovery and explanation read them.""" return semantic_shapes(project=project) @staticmethod def revalidation_request( - *, project: CompiledProject, model: CompiledModel, shapes: dict[str, dict[str, str]] + *, project: CompiledProjectView, model: CompiledModelView, shapes: dict[str, dict[str, str]] ) -> SqlSchemaValidationRequest: """Type recovery's request revalidating one failed model against `shapes`.""" @@ -59,7 +59,7 @@ def revalidation_request( @staticmethod def metadata_shapes( - *, project: CompiledProject, profile: ExpressionInferenceProfile + *, project: CompiledProjectView, profile: ExpressionInferenceProfile ) -> dict[str, dict[str, str]]: """Closed relation shapes, as the metadata checks read them under `profile`.""" @@ -69,35 +69,20 @@ def metadata_shapes( def column_names(value: object) -> tuple[str, ...]: """The column names a metadata config value lists.""" - return _names(value) - - @staticmethod - def audit_diagnostics( - *, - model: CompiledModel, - project: CompiledProject, - shape: dict[str, str], - shapes: dict[str, dict[str, str]], - profile: ExpressionInferenceProfile, - ) -> tuple[CompilerDiagnostic, ...]: - """The schema audit checks of one model with a closed shape.""" - - return _audit_errors( - model=model, project=project, shape=shape, shapes=shapes, profile=profile - ) + return names(value) @staticmethod def sql_test_columns( - *, project: CompiledProject, profile: ExpressionInferenceProfile + *, project: CompiledProjectView, profile: ExpressionInferenceProfile ) -> dict[str, tuple[str, ...]]: """The inferred output columns of every model SQL test CTE body.""" - return _sql_test_columns(project=project, profile=profile) + return sql_test_columns(project=project, profile=profile) @staticmethod def resource_sql_diagnostics( *, - project: CompiledProject, + project: CompiledProjectView, shapes: dict[str, dict[str, str]], profile: ExpressionInferenceProfile, ) -> tuple[CompilerDiagnostic, ...]: diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py b/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py index bdb609910d..139df54c06 100644 --- a/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py +++ b/src/sqlbuild/compiler/compile/classes/sql_test_cte_locator.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.sql_tests.helper_ctes import sql_test_cte_location +import sqlbuild._native as _native from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.spec.contracts.models import SourceLocation @@ -20,6 +20,13 @@ def locate( ) -> SourceLocation: """Locate a call inside one CTE of a SQL test block, else the CTE header, else the block.""" - return sql_test_cte_location( - test_file=test_file, test_block=test_block, cte_name=cte_name, call=call + line, column, end_line, end_column = _native.sql_test_cte_span( + test_file.contents, test_block.sql_body, cte_name, call + ) + return SourceLocation( + path=test_file.relative_path, + line=line, + column=column, + end_line=end_line, + end_column=end_column, ) diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py index 2e0cd9bd95..f13c663dde 100644 --- a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py +++ b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py @@ -1,85 +1,44 @@ -"""Per-file SQL-test scan results reused across compiles from the shared native store.""" +"""The compile's native SQL-test scan store, opened on first use and saved once.""" from __future__ import annotations import logging -from collections.abc import Callable, Sequence from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.compile.constants import ( - SQL_TEST_SCAN_STORE_FILE_NAME, - SQL_TEST_SCAN_STORE_VERSION, -) from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.profiling.main._metric import record_compile_metric -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class SqlTestScanCache: - """Whole per-file scan results keyed by installed code, scan algorithm, rules and text.""" + """The compile cache folder's scan store, keyed by the installed and native code.""" def __init__(self, *, cache_dir: Path | None) -> None: - self._path: Path | None = ( - None if cache_dir is None else cache_dir / SQL_TEST_SCAN_STORE_FILE_NAME - ) + self._cache_dir: Path | None = cache_dir self._store: _native.SqlTestScanStore | None = None - self._hits: int = 0 - self._misses: int = 0 - - def read[T]( - self, - *, - algorithm: str, - syntax: SqlLexicalSyntax, - parts: Sequence[str], - decode: Callable[[bytes], T | None], - ) -> T | None: - """Return one file's decoded stored result, or None when it must be scanned again.""" - - store: _native.SqlTestScanStore | None = self._opened() - if store is None: - return None - found: bytes | None = store.get([algorithm, syntax.cache_key, *parts]) - decoded: T | None = None if found is None else decode(found) - if decoded is None: - self._misses += 1 - else: - self._hits += 1 - return decoded - def write( - self, *, algorithm: str, syntax: SqlLexicalSyntax, parts: Sequence[str], value: bytes - ) -> None: - """Store one file's complete scan result; callers never store failed scans.""" + def opened(self) -> _native.SqlTestScanStore | None: + """The native store this compile reads and writes, or None when it has none.""" - store: _native.SqlTestScanStore | None = self._opened() - if store is not None: - store.put([algorithm, syntax.cache_key, *parts], value) + if self._cache_dir is None or self._store is not None: + return self._store + error: str | None + self._store, error = _native.SqlTestScanStore.open_in( + str(self._cache_dir), compiled_code_identity() + ) + if error is not None: + logging.getLogger(__name__).debug("SQL test scan store not loaded: %s", error) + self._cache_dir = None + return self._store def save(self) -> None: """Persist results stored by this compile and report hit and miss counts.""" - record_compile_metric(metric="sql_test_scan_cache_hits", value=self._hits) - record_compile_metric(metric="sql_test_scan_cache_misses", value=self._misses) - if self._store is None or self._path is None: + hits, misses = (0, 0) if self._store is None else self._store.counts() + record_compile_metric(metric="sql_test_scan_cache_hits", value=hits) + record_compile_metric(metric="sql_test_scan_cache_misses", value=misses) + if self._store is None: return - try: - _ = self._store.save(str(self._path)) - except (OSError, RuntimeError) as error: + _written, error = self._store.save() + if error is not None: logging.getLogger(__name__).debug("SQL test scan store not saved: %s", error) - - def _opened(self) -> _native.SqlTestScanStore | None: - if self._path is None: - return None - if self._store is None: - environment: str = _native.content_digest( - [SQL_TEST_SCAN_STORE_VERSION, compiled_code_identity()] - ) - try: - self._store = _native.SqlTestScanStore(str(self._path), environment) - except (OSError, RuntimeError) as error: - logging.getLogger(__name__).debug("SQL test scan store not loaded: %s", error) - self._path = None - return None - return self._store diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 1cd0e085bb..58e3475c16 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -2,24 +2,19 @@ from __future__ import annotations -import re - from sqlbuild.compiler.compile.classes.compile_input_read_registry import ( CompileInputReadRegistry, ) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.python_nodes.types import SqlResourceRefKind PRESERVE_TARGET_VALUE: str = "preserve" -SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE: str = "P004" ATTACHED_AUDIT_READS_OWN_DEPENDANT_CODE: str = "P005" -MACRO_GENERATED_REFERENCE_CODE: str = "P006" HOOK_READS_OWN_DEPENDANT_CODE: str = "P007" HARD_CODED_PROJECT_RELATION_CODE: str = "P008" UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE: str = "P009" -MISSING_DESCRIPTION_CODE: str = "P010" -CURSOR_MODEL_WITHOUT_INPUTS_CODE: str = "P011" REFERENCE_CALL_SYNTAX_CODE: str = "P012" SQL_TEST_HELPER_REFERENCE_CODE: str = "P013" DESCRIPTION_REQUIRED_INPUT_KINDS: dict[str, str] = { @@ -65,23 +60,10 @@ MIGRATE_FORCE_CONFIG_KEY: str = "migrate_force" COLUMN_MIGRATE_FROM_KEY: str = "migrate_from" MAX_MICROBATCHES_CONFIG_KEY: str = "max_microbatches" -MODEL_HEADER_METADATA_KEYS: frozenset[str] = frozenset( - { - "description", - "columns", - "dynamic_columns", - "model_schema", - "audits", - "audit_factories", - "enums", - "constants", - } -) SQL_OPEN_PAREN_TOKEN: str = "(" SQL_ARGUMENT_SEPARATOR_TOKEN: str = "," SQL_STATEMENT_TERMINATOR_TOKEN: str = ";" -SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`", "$"}) SQL_WITH_KEYWORD: str = "WITH" SQL_CEREMONIAL_SELECT_VALUE: str = "1" OMITTED_CEREMONIAL_SELECT_SQL: str = f"\nSELECT {SQL_CEREMONIAL_SELECT_VALUE}" @@ -92,19 +74,10 @@ MACRO_TOKEN: str = "@" DECLARATION_REFERENCE_NAMES: frozenset[str] = frozenset({"enum", "const"}) MACRO_CONTEXT_PARAMETER_NAME: str = "ctx" -MACRO_ARGUMENT_ERROR_TAG: str = "error" -MACRO_ARGUMENT_NESTED_CALL_TAG: str = "c" SQL_INTERPOLATION_TOKEN: str = "@@" TEMPLATE_OPEN_TOKEN: str = "${" -MACRO_CALL_PATTERN: re.Pattern[str] = re.compile(r"@[A-Za-z_][A-Za-z0-9_]*\s*\(") -SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN: re.Pattern[str] = re.compile( - r"(?[A-Za-z_][A-Za-z0-9_]*)'" -) -SQL_ARGUMENT_RAW_PARAMETER_PATTERN: re.Pattern[str] = re.compile( - r"(?[A-Za-z_][A-Za-z0-9_]*)(?![A-Za-z0-9_])(?!\s*\()" -) EXPECTED_TEST_CTE_PREFIX: str = "__expected__" ASSERT_TEST_CTE_PREFIX: str = "__assert__" REF_TEST_CTE_PREFIX: str = SqlReferenceKind.REF.fixture_cte_prefix @@ -123,8 +96,12 @@ COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" COMPILE_INPUT_READS: CompileInputReadRegistry = CompileInputReadRegistry() -SQL_TEST_SCAN_STORE_FILE_NAME: str = "sql-test-scans.bin" RETIRED_FACT_CACHE_DIRECTORY_NAME: str = "facts-v1" -SQL_TEST_SCAN_STORE_VERSION: str = "sql-test-scan-store-v1" +RETIRED_REFERENCE_CACHE_DIRECTORY_NAME: str = "references-v3" SQL_TEST_CTE_SCAN_ALGORITHM: str = "expanded-sql-test-ctes-v1" SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: str = "sql-test-expected-models-v1" +EXPLICIT_REFERENCE_KINDS: dict[str, SqlResourceRefKind] = { + SqlReferenceKind.REF.value: SqlResourceRefKind.MODEL, + SqlReferenceKind.SOURCE.value: SqlResourceRefKind.SOURCE, + SqlReferenceKind.SEED.value: SqlResourceRefKind.SEED, +} diff --git a/src/sqlbuild/compiler/compile/exceptions.py b/src/sqlbuild/compiler/compile/exceptions.py index bf527cee32..3581348c65 100644 --- a/src/sqlbuild/compiler/compile/exceptions.py +++ b/src/sqlbuild/compiler/compile/exceptions.py @@ -99,10 +99,6 @@ class MacroDeclarationLookupError(CompileInputError, KeyError): """Retain authored diagnostics while honoring the Mapping lookup contract.""" -class NativeSqlTestResponseError(CompileInputError): - """Raised when native SQL-test extraction returns malformed data.""" - - class SqlTestExtractionError(CompileInputError): """Raised when a SQL test cannot be split into CTEs; names the test and offending text.""" diff --git a/src/sqlbuild/compiler/compile/main/_assemble_project.py b/src/sqlbuild/compiler/compile/main/_assemble_project.py index a165f9c226..1c255587cd 100644 --- a/src/sqlbuild/compiler/compile/main/_assemble_project.py +++ b/src/sqlbuild/compiler/compile/main/_assemble_project.py @@ -3,10 +3,10 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import replace from functools import partial from pathlib import Path +import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( @@ -16,10 +16,11 @@ explicit_reference_report, ) from sqlbuild.compiler.compile.models import ( - CompiledProject, - CompileProjectInputs, PythonSqlReferenceReport, ) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView +from sqlbuild.compiler.compiled_project.main._record_model_analyses import record_model_analyses +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge from sqlbuild.compiler.macro_bridge.main.run_with_macro_bridge import run_with_macro_bridge @@ -28,16 +29,16 @@ @with_collected_compile_diagnostics def assemble_project( *, - inputs: CompileProjectInputs, + inputs: CompileInputsView, inference_profile: ExpressionInferenceProfile | None = None, skip_column_inference: bool = False, column_lineage_mode: ColumnLineageMode = ColumnLineageMode.FAST, analysis_cache_dir: Path | None = None, analysis_model_names: frozenset[str] | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """Convert compile inputs into the planner-ready project view.""" - assemble: Callable[[], CompiledProject] = partial( + assemble: Callable[[], CompiledProjectView] = partial( assemble_compiled_project, inputs=inputs, inference_profile=inference_profile, @@ -46,8 +47,11 @@ def assemble_project( analysis_cache_dir=analysis_cache_dir, analysis_model_names=analysis_model_names, ) - project: CompiledProject = ( - run_with_macro_bridge(stage=assemble) if active_macro_bridge() is None else assemble() + project: CompiledProjectView = _with_native_project( + project=( + run_with_macro_bridge(stage=assemble) if active_macro_bridge() is None else assemble() + ), + native_project=inputs.native_project, ) references: PythonSqlReferenceReport = explicit_reference_report( project=project, @@ -56,8 +60,21 @@ def assemble_project( ) if not references.diagnostics and not references.unmatched: return project - return replace( - project, + return project_with( + project=project, diagnostics=(*project.diagnostics, *references.diagnostics), unmatched_literal_sql_relations=references.unmatched, ) + + +def _with_native_project( + *, project: CompiledProjectView, native_project: _native.NativeCompiledProject | None +) -> CompiledProjectView: + """Hand the compile's native project the assembled analyses and keep it on the project.""" + + if native_project is None: + return project + _ = record_model_analyses(project=native_project, models=project.models) + if project.native_project is native_project: + return project + return project_with(project=project, native_project=native_project) diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 819da6a2e5..4505b74dd0 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -4,37 +4,19 @@ from dataclasses import replace from pathlib import Path +from typing import Any -from sqlbuild.compiler.compile._helpers.attachment.audits import build_project_audit_inputs +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.attachment.core import ( build_effective_connection, build_effective_settings, build_effective_vars, build_model_inputs, - build_seed_inputs, ) from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope -from sqlbuild.compiler.compile._helpers.attachment.functions import build_sql_function_inputs -from sqlbuild.compiler.compile._helpers.attachment.references import ( - validate_table_function_call_arities, -) -from sqlbuild.compiler.compile._helpers.attachment.sources import build_source_inputs -from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( - build_scenario_inputs, - build_test_inputs_with_cache, -) -from sqlbuild.compiler.compile._helpers.audit_factories.core import ( - build_audit_factory_orphan_diagnostics, -) -from sqlbuild.compiler.compile._helpers.config.cursor_inputs import ( - cursor_model_without_inputs_diagnostics, -) from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( with_collected_compile_diagnostics, ) -from sqlbuild.compiler.compile._helpers.diagnostics.descriptions import ( - missing_description_diagnostics, -) from sqlbuild.compiler.compile._helpers.macro_bridge.call_store import ( target_context_with_macro_call_store, ) @@ -46,16 +28,9 @@ from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros from sqlbuild.compiler.compile.models import ( CompileAdapterContext, - CompileAuditInput, - CompileModelInput, - CompileProjectInputs, CompilerDiagnostic, - CompileSeedInput, - CompileSourceInput, - CompileSqlFunctionInput, CompileSqlScenarioInput, CompileSqlTestInput, - DeclarationExpansionContext, DeclarationResolutionContext, DeclarationScopeBuild, LoadedMacro, @@ -63,6 +38,17 @@ ModelInputBuildContext, ModelInputScopeBuild, ) +from sqlbuild.compiler.compile.types import ( + AuditInput, + CompileInputsView, + FunctionInput, + ModelInput, + SeedInput, + SourceInput, +) +from sqlbuild.compiler.compiled_project.main._attach_checks import attach_checks +from sqlbuild.compiler.compiled_project.main._attach_resources import attach_resources +from sqlbuild.compiler.compiled_project.main._compile_inputs_view import compile_inputs_view from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, DiscoveredProjectInputs, @@ -89,7 +75,7 @@ def build_compile_inputs( external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, resolved_connection: dict[str, object] | None = None, no_cache: bool = False, -) -> CompileProjectInputs: +) -> CompileInputsView: """Attach discovered metadata into the first compile input snapshot.""" effective_target_name: str | None @@ -152,80 +138,49 @@ def build_compile_inputs( no_sql_validation=no_sql_validation, defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, - reference_cache_dir=compile_cache_dir, ) model_context = model_build.context - seed_inputs: tuple[CompileSeedInput, ...] = build_seed_inputs(discovered_inputs) - sql_function_inputs: tuple[CompileSqlFunctionInput, ...] = _build_sql_functions( + seed_inputs: tuple[SeedInput, ...] + sql_function_inputs: tuple[FunctionInput, ...] + source_inputs: tuple[SourceInput, ...] + seed_inputs, sql_function_inputs, source_inputs = attach_resources( discovered_inputs=discovered_inputs, - context=model_context, - declaration_expansion=model_context.declaration_expansion, - model_inputs=model_build.inputs, + native_project=model_build.native_project, + settings=( + model_context.effective_settings, + model_context.target_config, + model_context.macro_context.adapter_name, + ), no_sql_validation=no_sql_validation, python_functions_inherit_default_namespace=( adapter_context.python_functions_inherit_default_namespace ), ) - source_inputs: tuple[CompileSourceInput, ...] = build_source_inputs( - discovered_inputs=discovered_inputs, - effective_vars=model_context.effective_vars, - effective_settings=model_context.effective_settings, - macro_context=model_context.macro_context, - loaded_macros=model_context.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - no_sql_validation=no_sql_validation, - ) - test_inputs: tuple[CompileSqlTestInput, ...] = build_test_inputs_with_cache( + test_inputs: tuple[CompileSqlTestInput, ...] + scenario_inputs: tuple[CompileSqlScenarioInput, ...] + audit_inputs: tuple[AuditInput, ...] + diagnostics: tuple[CompilerDiagnostic, ...] + test_inputs, scenario_inputs, audit_inputs, diagnostics = attach_checks( discovered_inputs=discovered_inputs, - effective_vars=model_context.effective_vars, - macro_context=model_context.macro_context, - loaded_macros=model_context.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_function_inputs=sql_function_inputs, - sql_lexical_syntax=model_context.sql_lexical_syntax, + native_project=model_build.native_project, + settings=( + model_context.effective_settings, + adapter_context.value_renderer, + model_context.sql_lexical_syntax, + ), scan_cache=declaration_scope.sql_test_scans, - ) - scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( - discovered_inputs=discovered_inputs, - effective_vars=model_context.effective_vars, - macro_context=model_context.macro_context, - loaded_macros=model_context.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_lexical_syntax=model_context.sql_lexical_syntax, - ) - audit_inputs: tuple[CompileAuditInput, ...] - audit_diagnostics: tuple[CompilerDiagnostic, ...] - audit_inputs, audit_diagnostics = build_project_audit_inputs( - discovered_inputs=discovered_inputs, - context=model_context, - model_inputs=model_build.inputs, source_inputs=source_inputs, - seed_inputs=seed_inputs, - ) - diagnostics: tuple[CompilerDiagnostic, ...] = ( - *audit_diagnostics, - *model_build.diagnostics, - *missing_description_diagnostics( - discovered_inputs=discovered_inputs, - model_inputs=model_build.inputs, - seed_inputs=seed_inputs, - source_inputs=source_inputs, - function_inputs=sql_function_inputs, - ), - *cursor_model_without_inputs_diagnostics(model_inputs=model_build.inputs), ) - return CompileProjectInputs( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - discovered_inputs=discovered_inputs, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, - run_id=resolved_run_id, - effective_target_name=effective_target_name, - effective_target=effective_target, - compile_cache_dir=compile_cache_dir, - effective_connection=( + fields: dict[str, Any] = { + "project_config": discovered_inputs.project_config, + "local_config": discovered_inputs.local_config, + "discovered_inputs": discovered_inputs, + "sql_lexical_syntax": adapter_context.sql_lexical_syntax, + "run_id": resolved_run_id, + "effective_target_name": effective_target_name, + "effective_target": effective_target, + "compile_cache_dir": compile_cache_dir, + "effective_connection": ( resolved_connection if resolved_connection is not None else build_effective_connection( @@ -235,53 +190,26 @@ def build_compile_inputs( effective_vars=effective_vars, ) ), - effective_settings=effective_settings, - no_sql_validation=no_sql_validation, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - public_enums=model_build.declarations.enums, - public_constants=model_build.declarations.constants, - model_inputs=model_build.inputs, - seed_inputs=seed_inputs, - source_inputs=source_inputs, - sql_function_inputs=sql_function_inputs, - test_inputs=test_inputs, - scenario_inputs=scenario_inputs, - audit_inputs=audit_inputs, - diagnostics=diagnostics, - external_sql_reference_resolver=external_sql_reference_resolver, - scope_index=declaration_scope.index, - declaration_scope=declaration_scope, - ) - - -def _build_sql_functions( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - declaration_expansion: DeclarationExpansionContext, - model_inputs: tuple[CompileModelInput, ...], - no_sql_validation: bool, - python_functions_inherit_default_namespace: bool, -) -> tuple[CompileSqlFunctionInput, ...]: - sql_function_inputs: tuple[CompileSqlFunctionInput, ...] = build_sql_function_inputs( - discovered_inputs=discovered_inputs, - effective_vars=context.effective_vars, - effective_settings=context.effective_settings, - target_config=context.target_config, - macro_context=context.macro_context, - loaded_macros=context.loaded_macros, - declaration_expansion=declaration_expansion, - no_sql_validation=no_sql_validation, - python_functions_inherit_default_namespace=python_functions_inherit_default_namespace, - sql_lexical_syntax=context.sql_lexical_syntax, - ) - validate_table_function_call_arities( - model_inputs=model_inputs, - sql_function_inputs=sql_function_inputs, - ) - return sql_function_inputs + "effective_settings": effective_settings, + "no_sql_validation": no_sql_validation, + "effective_vars": effective_vars, + "macro_context": macro_context, + "loaded_macros": declaration_scope.loaded_macros, + "public_enums": model_build.declarations.enums, + "public_constants": model_build.declarations.constants, + "model_inputs": model_build.inputs, + "seed_inputs": seed_inputs, + "source_inputs": source_inputs, + "sql_function_inputs": sql_function_inputs, + "test_inputs": test_inputs, + "scenario_inputs": scenario_inputs, + "audit_inputs": audit_inputs, + "diagnostics": diagnostics, + "external_sql_reference_resolver": external_sql_reference_resolver, + "scope_index": declaration_scope.index, + "declaration_scope": declaration_scope, + } + return compile_inputs_view(fields=fields, native_project=model_build.native_project) def _build_models_with_declarations( @@ -291,7 +219,6 @@ def _build_models_with_declarations( no_sql_validation: bool, defer_model_sql_validation: bool, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - reference_cache_dir: Path | None, ) -> ModelInputScopeBuild: public_enums: dict[str, EnumDeclaration] public_constants: dict[str, ConstantDeclaration] @@ -305,7 +232,8 @@ def _build_models_with_declarations( enums=public_enums, constants=public_constants, ) - model_inputs: tuple[CompileModelInput, ...] = build_model_inputs( + native_project: _native.NativeCompiledProject = _native.NativeCompiledProject() + model_inputs: tuple[ModelInput, ...] = build_model_inputs( discovered_inputs=discovered_inputs, context=replace( context, @@ -316,15 +244,15 @@ def _build_models_with_declarations( no_sql_validation=no_sql_validation, defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, - reference_cache_dir=reference_cache_dir, + native_project=native_project, ) return ModelInputScopeBuild( inputs=model_inputs, + native_project=native_project, declarations=declarations, context=replace( context, public_enums=declarations.enums, public_constants=declarations.constants, ), - diagnostics=build_audit_factory_orphan_diagnostics(discovered_inputs=discovered_inputs), ) diff --git a/src/sqlbuild/compiler/compile/main/_cursor_roles.py b/src/sqlbuild/compiler/compile/main/_cursor_roles.py index 956b85fcec..07e75bb5f4 100644 --- a/src/sqlbuild/compiler/compile/main/_cursor_roles.py +++ b/src/sqlbuild/compiler/compile/main/_cursor_roles.py @@ -5,11 +5,12 @@ from typing import Any, cast from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY -from sqlbuild.compiler.compile.models import CompiledModel, CursorInputRoles +from sqlbuild.compiler.compile.models import CursorInputRoles +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner.types import CursorInputRole, MicrobatchStrategy -def resolve_cursor_input_roles(*, model: CompiledModel) -> CursorInputRoles: +def resolve_cursor_input_roles(*, model: CompiledModelView) -> CursorInputRoles: """Resolve authored cursor input fields into explicit effective roles.""" authored: object | None = model.config.values.get(CURSOR_INPUTS_CONFIG_KEY) diff --git a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py index a06cf6e23b..60e92e5e21 100644 --- a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py +++ b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py @@ -2,24 +2,17 @@ from __future__ import annotations -from dataclasses import replace - from sqlbuild.compiler.compile._helpers.named_declarations.model_schemas import ( model_schema_scope_usages, ) -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.scopes.main._get_placement_validated_scope_index import ( get_placement_validated_scope_index, ) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - ScopeIndex, - UsageRecord, -) +from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord -def scope_index_with_compile_usages(*, inputs: CompileProjectInputs) -> ScopeIndex: +def scope_index_with_compile_usages(*, inputs: CompileInputsView) -> ScopeIndex: """Return compile inputs' canonical index with all usage and placement facts.""" collected_usages: list[UsageRecord] = list( @@ -43,38 +36,8 @@ def scope_index_with_compile_usages(*, inputs: CompileProjectInputs) -> ScopeInd collected_usages.extend(test_input.declaration_usages) for scenario_input in inputs.scenario_inputs: collected_usages.extend(scenario_input.declaration_usages) - usages: tuple[UsageRecord, ...] = tuple(dict.fromkeys(collected_usages)) - dependencies: dict[DeclarationIdentity, list[DeclarationIdentity]] = {} - for usage in usages: - if isinstance(usage.consumer, DeclarationIdentity): - dependencies.setdefault(usage.consumer, []).append(usage.declaration) - declarations: tuple[DeclarationRecord, ...] = tuple( - replace( - record, - macro=replace( - record.macro, - dependencies=tuple( - dict.fromkeys( - (*record.macro.dependencies, *dependencies.get(record.identity, ())) - ) - ), - ), - ) - if record.macro is not None - else record - for record in inputs.scope_index.declarations - ) - index: ScopeIndex = replace( - inputs.scope_index, - declarations=declarations, - usages=tuple(dict.fromkeys((*inputs.scope_index.usages, *usages))), - completeness=replace( - inputs.scope_index.completeness, - runtime_usage=True, - promotion_impact=True, - ), - ) return get_placement_validated_scope_index( - index=index, + index=inputs.scope_index, enforce_placement=inputs.project_config.scopes.enforce_placement, + compile_usages=tuple(collected_usages), ) diff --git a/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py b/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py deleted file mode 100644 index f6d7255467..0000000000 --- a/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Compiler extraction of test and scenario declaration grants.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile._helpers.attachment.scope_relationships import ( - build_scope_relationship_grants as _build_scope_relationship_grants, -) -from sqlbuild.compiler.compile.models import ScopeRelationshipBuild -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def build_scope_relationship_grants( - *, - discovered_inputs: DiscoveredProjectInputs, - index: ScopeIndex, - sql_lexical_syntax: SqlLexicalSyntax, -) -> ScopeRelationshipBuild: - """Return expected-model grants while retaining independent extraction faults.""" - - return _build_scope_relationship_grants( - discovered_inputs=discovered_inputs, - index=index, - sql_lexical_syntax=sql_lexical_syntax, - ) diff --git a/src/sqlbuild/compiler/compile/main/_source_bindings.py b/src/sqlbuild/compiler/compile/main/_source_bindings.py index 478fd49b03..1f9bb0da27 100644 --- a/src/sqlbuild/compiler/compile/main/_source_bindings.py +++ b/src/sqlbuild/compiler/compile/main/_source_bindings.py @@ -4,12 +4,13 @@ from sqlbuild.compiler.compile._helpers.assembly.source_bindings import ( get_source_binding_diagnostics as _get_source_binding_diagnostics, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView def get_source_binding_diagnostics( *, - project: CompiledProject, + project: CompiledProjectView, columns: dict[str, tuple[ColumnInfo, ...]], profile: ExpressionInferenceProfile, selected_keys: frozenset[CompiledObjectKey], diff --git a/src/sqlbuild/compiler/compile/main/semantic_coverage.py b/src/sqlbuild/compiler/compile/main/semantic_coverage.py index 33b28784a6..a7c2ac9483 100644 --- a/src/sqlbuild/compiler/compile/main/semantic_coverage.py +++ b/src/sqlbuild/compiler/compile/main/semantic_coverage.py @@ -1,9 +1,9 @@ """Public semantic-check coverage report.""" from sqlbuild.compiler.compile._helpers.assembly.semantic_coverage import semantic_coverage -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView -def get_semantic_coverage(project: CompiledProject) -> dict[str, tuple[str, ...]]: +def get_semantic_coverage(project: CompiledProjectView) -> dict[str, tuple[str, ...]]: """Return per-model partial-check reasons.""" return semantic_coverage(project) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 86ecc786ca..997f0f6de3 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -12,18 +12,27 @@ import sqlbuild._native as _native from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity -from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, + AuditInput, CompactBatchResponseCallback, + CompiledAuditView, + CompiledFunctionView, + CompiledModelView, CompiledResourceType, + CompiledSeedView, + CompiledSourceView, CompileInputsReadyCallback, DiagnosticPhase, DiagnosticSeverity, + FunctionInput, FunctionLanguage, + ModelInput, + SeedInput, + SourceInput, SqlTestMode, TypedSqlValueRenderer, ) @@ -59,7 +68,6 @@ from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, - GrantRecord, ResourceIdentity, ScopeIndex, ScopeLookup, @@ -67,7 +75,7 @@ VisibilityRecord, ) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlLexicalSyntax -from sqlbuild.python_nodes.models import SqlResourceRef +from sqlbuild.lint.types import RelationKeyRow from sqlbuild.spec.contracts.models import ( LocalConfig, ProjectConfig, @@ -281,22 +289,6 @@ class StaticMacroInventory: faults: tuple[StaticMacroFault, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class ScopeRelationshipFault: - """One test or scenario relationship extraction fault.""" - - relative_path: Path - message: str - - -@dataclass(frozen=True) -class ScopeRelationshipBuild: - """Expected-model grants and independently retained relationship faults.""" - - grants: tuple[GrantRecord, ...] = field(default_factory=tuple) - faults: tuple[ScopeRelationshipFault, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class DeclarationRuntimeProjection: """Process-only declaration values keyed by canonical static identity.""" @@ -334,15 +326,6 @@ class DeclarationScopeBuild: sql_test_scans: SqlTestScanCache -@dataclass(frozen=True) -class ParsedMacroArguments: - """One macro call's argument objects and the typed references written in them, in order.""" - - args: tuple[object, ...] - kwargs: dict[str, object] - typed_references: tuple[SqlResourceRef, ...] - - @dataclass(frozen=True) class MacroContext: """Expose target settings and caller-visible declarations to a Python SQL macro.""" @@ -396,19 +379,6 @@ def render_enum_member(self, *, enum_name: str, member_name: str) -> str: return render_enum_member_value(value=value) -@dataclass(frozen=True) -class MacroExpansionState: - """Inputs and recorded facts of one authored string's macro expansion.""" - - loaded_macros: dict[str, LoadedMacro] - macro_overrides: dict[str, str] - macro_context: MacroContext - declaration_resolver: DeclarationScopeResolver | None - declarations: DeclarationResolutionContext | None - facts: MacroExpansionFacts - consumer: ResourceIdentity | DeclarationIdentity | None = None - - @dataclass(frozen=True) class MacroExpansionResult: """Expanded SQL and resolved macro usage facts from one authored string.""" @@ -438,15 +408,6 @@ class AuthoredSqlExpansionResult: argument_references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class HookExpansionResult: - """Expanded model hook values and declaration usage facts.""" - - values: dict[str, object] - usages: tuple[UsageRecord, ...] = field(default_factory=tuple) - references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class DeclarationResolutionContext: """Declarations visible during one authored SQL expansion.""" @@ -522,10 +483,10 @@ def declaration_expansion(self) -> DeclarationExpansionContext: class ModelInputScopeBuild: """Model inputs paired with public declaration artifacts and updated context.""" - inputs: tuple[CompileModelInput, ...] + inputs: tuple[ModelInput, ...] declarations: DeclarationResolutionContext context: ModelInputBuildContext - diagnostics: tuple[CompilerDiagnostic, ...] = () + native_project: _native.NativeCompiledProject = field(repr=False, compare=False) @dataclass(frozen=True) @@ -748,21 +709,6 @@ class CompileSqlScenarioCte: sql_body: str -@dataclass(frozen=True) -class CompileSqlScenarioCtes: - """Extracted top-level SQL-native scenario CTE semantics.""" - - authored_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - expected_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - assertion_ctes: tuple[CompileSqlScenarioCte, ...] = field(default_factory=tuple) - source_fixture_names: tuple[str, ...] = field(default_factory=tuple) - ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - seed_fixture_names: tuple[str, ...] = field(default_factory=tuple) - dbt_ref_fixture_names: tuple[str, ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_names: tuple[str, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class FunctionArgument: """One SQL function argument with an adapter-native type string.""" @@ -806,6 +752,20 @@ class CompileSqlFunctionInput: tags: tuple[str, ...] = field(default_factory=tuple) description: str | None = None + @property + def assembly_row(self) -> tuple[str, str, list[str], list[tuple[str, str, str | None]]]: + """`(node type, name, tags, references)`, what assembly resolves dependencies from.""" + + return ( + CompiledResourceType.TABLE_FN if self.return_columns else CompiledResourceType.UDF, + self.name, + list(self.tags), + [ + (str(reference.ref_kind), reference.ref_name, reference.ref_package) + for reference in self.references + ], + ) + @dataclass(frozen=True) class CompileModelInput: @@ -826,7 +786,52 @@ class CompileModelInput: macro_deps: tuple[str, ...] = field(default_factory=tuple) macro_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) + + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: + """Each reference as `(kind, name, package, call argument count)`.""" + + return [ + (str(ref.ref_kind), ref.ref_name, ref.ref_package, ref.call_argument_count) + for ref in self.references + ] + + @property + def relation_key_row(self) -> RelationKeyRow: + """The model's declared key facts, for deriving its unique keys.""" + + return _native.relation_key_row( + "ref", self.model_file.file_path.stem, self.config.values, self.schema_entry + ) + + @property + def attached_schema_name(self) -> str | None: + """The name of the schema entry attached to the model, or `None`.""" + + return self.schema_entry.name if self.schema_entry is not None else None + sql_expansion: CompiledSqlExpansion | None = field(default=None, compare=False, repr=False) + lint_expansion: CompiledSqlExpansion | None = field(default=None, compare=False, repr=False) + """The compile's whole expansion, variables included, which early SQL lint reuses.""" + + +@dataclass(frozen=True) +class HandBuiltAnalysis: + """A hand-built model input's session position, fallback columns and standalone proof.""" + + session: tuple[Any, int] | None + fallback: list[tuple[str, str | None, str]] | None + standalone_proof: ( + tuple[ + bool, + list[tuple[str, str | None, str]], + list[tuple[str, str | None]], + list[str], + str | None, + bool, + ] + | None + ) @dataclass(frozen=True) @@ -846,7 +851,7 @@ class ModelSqlAnalysis: class ModelSqlAnalysisRequest: """Inputs to one model's SQL analysis.""" - model_input: CompileModelInput + model_input: ModelInput query_sql: str placeholders: dict[str, str] | None cache_key: str | None @@ -870,6 +875,13 @@ class CompileSourceInput: source_file: DiscoveredSourceFile declaration_usages: tuple[UsageRecord, ...] = field(default_factory=tuple) + @property + def assembly_row(self) -> tuple[str, bool, str | None, str | None]: + """`(name, has a loader, database, schema)`, what assembly resolves a namespace from.""" + + entry: SourceEntry = self.source_entry + return (entry.name, entry.loader is not None, entry.database, entry.schema) + @dataclass(frozen=True) class CompileSqlScenarioInput: @@ -924,6 +936,22 @@ def __post_init__(self) -> None: AttachedAuditTargetKind(self.attached_target_kind), ) + @property + def assembly_row( + self, + ) -> tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]: + """`(references, attached (kind, name))`, what assembly resolves scope deps from.""" + + return ( + [ + (str(reference.ref_kind), reference.ref_name, reference.ref_package) + for reference in self.references + ], + (str(self.attached_target_kind), self.attached_target_name) + if self.attached_target_kind is not None and self.attached_target_name is not None + else None, + ) + @dataclass(frozen=True) class CompileProjectInputs: @@ -945,17 +973,20 @@ class CompileProjectInputs: loaded_macros: dict[str, LoadedMacro] = field(default_factory=dict) public_enums: dict[str, EnumDeclaration] = field(default_factory=dict) public_constants: dict[str, ConstantDeclaration] = field(default_factory=dict) - model_inputs: tuple[CompileModelInput, ...] = field(default_factory=tuple) - seed_inputs: tuple[CompileSeedInput, ...] = field(default_factory=tuple) - source_inputs: tuple[CompileSourceInput, ...] = field(default_factory=tuple) - sql_function_inputs: tuple[CompileSqlFunctionInput, ...] = field(default_factory=tuple) + model_inputs: tuple[ModelInput, ...] = field(default_factory=tuple) + seed_inputs: tuple[SeedInput, ...] = field(default_factory=tuple) + source_inputs: tuple[SourceInput, ...] = field(default_factory=tuple) + sql_function_inputs: tuple[FunctionInput, ...] = field(default_factory=tuple) test_inputs: tuple[CompileSqlTestInput, ...] = field(default_factory=tuple) scenario_inputs: tuple[CompileSqlScenarioInput, ...] = field(default_factory=tuple) - audit_inputs: tuple[CompileAuditInput, ...] = field(default_factory=tuple) + audit_inputs: tuple[AuditInput, ...] = field(default_factory=tuple) diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None scope_index: ScopeIndex = field(default_factory=ScopeIndex) declaration_scope: DeclarationScopeBuild | None = field(default=None, repr=False, compare=False) + native_project: _native.NativeCompiledProject | None = field( + default=None, repr=False, compare=False + ) @dataclass(frozen=True) @@ -1001,6 +1032,71 @@ class CompiledModel: unchecked_output_columns: frozenset[str] = frozenset() rejected_sql_analysis_opt_out: SourceLocation | None = None + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: + """Each reference as `(kind, name, package, call argument count)`.""" + + return [ + (str(ref.ref_kind), ref.ref_name, ref.ref_package, ref.call_argument_count) + for ref in self.references + ] + + @property + def relation_key_row(self) -> RelationKeyRow: + """The model's declared key facts, for deriving its unique keys.""" + + return _native.relation_key_row("ref", self.name, self.config.values, self.schema_entry) + + @property + def key_row(self) -> tuple[str, str]: + """`(resource type, name)` of the model's key.""" + + return (str(self.key.resource_type), self.key.name) + + @property + def dep_rows(self) -> list[tuple[str, str]]: + """`(resource type, name)` of each dependency.""" + + return [(str(dep.resource_type), dep.name) for dep in self.deps] + + @property + def destination_row(self) -> tuple[str | None, str | None, str]: + """`(database, schema, name)` of the model's destination.""" + + return (self.destination.database, self.destination.schema, self.destination.name) + + @property + def inferred_rows(self) -> list[tuple[str, str | None, str]] | None: + """Each inferred column as `(name, type, nullability)`, or `None`.""" + + if self.inferred_columns is None: + return None + return [ + (column.name, column.type, str(column.nullability)) for column in self.inferred_columns + ] + + @property + def schema_column_rows(self) -> list[tuple[str, str | None]] | None: + """Each declared column of the attached schema as `(name, type)`, or `None`.""" + + if self.schema_entry is None: + return None + return [(column.name, column.type) for column in self.schema_entry.columns] + + @property + def has_dynamic_columns(self) -> bool: + """Whether the attached schema declares dynamic columns.""" + + return self.schema_entry is not None and bool(self.schema_entry.dynamic_columns) + + @property + def output_column_count(self) -> int: + """Inferred columns, else the attached schema's declared columns, else zero.""" + + if self.inferred_columns is not None: + return len(self.inferred_columns) + return len(self.schema_entry.columns) if self.schema_entry is not None else 0 + @dataclass(frozen=True) class CompiledSource: @@ -1140,11 +1236,11 @@ class CompiledProject: compile_cache_dir: Path | None = None settings: SettingsConfig = field(default_factory=SettingsConfig) scenario: ScenarioConfig = field(default_factory=ScenarioConfig) - models: tuple[CompiledModel, ...] = field(default_factory=tuple) - sources: tuple[CompiledSource, ...] = field(default_factory=tuple) - seeds: tuple[CompiledSeed, ...] = field(default_factory=tuple) - functions: tuple[CompiledFunction, ...] = field(default_factory=tuple) - audits: tuple[CompiledAudit, ...] = field(default_factory=tuple) + models: tuple[CompiledModelView, ...] = field(default_factory=tuple) + sources: tuple[CompiledSourceView, ...] = field(default_factory=tuple) + seeds: tuple[CompiledSeedView, ...] = field(default_factory=tuple) + functions: tuple[CompiledFunctionView, ...] = field(default_factory=tuple) + audits: tuple[CompiledAuditView, ...] = field(default_factory=tuple) sql_tests: tuple[CompiledSqlTest, ...] = field(default_factory=tuple) sql_scenarios: tuple[CompiledSqlScenario, ...] = field(default_factory=tuple) loader_functions: tuple[DiscoveredLoaderFunction, ...] = field(default_factory=tuple) @@ -1163,49 +1259,68 @@ class CompiledProject: default_factory=dict, compare=False, repr=False ) native_session: Any | None = field(default=None, compare=False, repr=False) + native_project: _native.NativeCompiledProject | None = field( + default=None, compare=False, repr=False + ) + @cached_property + def lineage_graph(self) -> _native.NativeProjectGraph: + """The lineage edges, tags, model folders and names, indexed natively once per project.""" + + rows: list[tuple[tuple[str, str], list[tuple[str, str]], list[str], str | None]] = [ + ( + self._graph_key(model.key), + self._graph_keys(model.deps), + self._graph_tags(model.config.values.get("tags")), + str(model.relative_path.parent), + ) + for model in self.models + ] + rows.extend( + (self._graph_key(source.key), self._graph_keys(source.deps), [], None) + for source in self.sources + ) + rows.extend( + ( + self._graph_key(seed.key), + self._graph_keys(seed.deps), + list(seed.schema_entry.tags), + None, + ) + for seed in self.seeds + ) + rows.extend( + ( + self._graph_key(function.key), + self._graph_keys(function.deps), + list(function.tags), + None, + ) + for function in self.functions + ) + return _native.NativeProjectGraph.from_resources(rows) -@dataclass(frozen=True) -class CompileSqlTestCte: - """One top-level SQL-native test CTE extracted after macro expansion.""" - - name: str - sql_body: str - - -@dataclass(frozen=True) -class CompileModelSqlTestCtes: - """Extracted model-mode SQL-native test CTE semantics.""" - - authored_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - macro_mocks: dict[str, str] = field(default_factory=dict) - mock_model_names: tuple[str, ...] = field(default_factory=tuple) - mock_source_names: tuple[str, ...] = field(default_factory=tuple) - mock_seed_names: tuple[str, ...] = field(default_factory=tuple) - mock_dbt_ref_names: tuple[str, ...] = field(default_factory=tuple) - mock_table_function_names: tuple[str, ...] = field(default_factory=tuple) - expected_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - expected_model_names: tuple[str, ...] = field(default_factory=tuple) - assertion_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) - assertion_names: tuple[str, ...] = field(default_factory=tuple) - + @staticmethod + def _graph_key(key: CompiledObjectKey) -> tuple[str, str]: + return (str(key.resource_type), key.name) -@dataclass(frozen=True) -class CompileDirectLogicSqlTestCtes: - """Extracted direct-logic SQL-native test CTE semantics.""" + @classmethod + def _graph_keys(cls, keys: Sequence[CompiledObjectKey]) -> list[tuple[str, str]]: + return [cls._graph_key(key) for key in keys] - mode: SqlTestMode - helper_ctes: tuple[CompileSqlTestCte, ...] - actual_cte: CompileSqlTestCte - expected_cte: CompileSqlTestCte + @staticmethod + def _graph_tags(value: object) -> list[str]: + if isinstance(value, list | tuple): + return [str(item) for item in value] + return [] @dataclass(frozen=True) -class CompileSqlTestCtes: - """Extracted top-level SQL-native test CTE semantics.""" +class CompileSqlTestCte: + """One top-level SQL-native test CTE extracted after macro expansion.""" - mode: SqlTestMode - payload: CompileModelSqlTestCtes | CompileDirectLogicSqlTestCtes + name: str + sql_body: str @dataclass(frozen=True) @@ -1228,24 +1343,6 @@ class CompileModelSqlTestInputPayload: reference_target_model_names: tuple[str, ...] = field(default_factory=tuple) -@dataclass(frozen=True) -class SqlTestReads: - """Helpers one SQL test reads and the unmocked models they and its expected rows reference.""" - - read_helper_names: tuple[str, ...] - reference_target_model_names: tuple[str, ...] - - -@dataclass(frozen=True) -class SqlTestCteGraph: - """Which authored helper and mock CTEs one SQL test reads, keyed by case-folded name.""" - - ctes: dict[str, CompileSqlTestCte] - reads: dict[str, tuple[str, ...]] - parsed_reads: dict[str, tuple[str, ...]] - reader_reads: tuple[str, ...] - - @dataclass(frozen=True) class CompileDirectLogicSqlTestInputPayload: """Direct-logic SQL test compile payload.""" @@ -1627,16 +1724,3 @@ class ModelValidatorContext: settings: SettingsConfig external_sql_reference_resolver: ExternalSqlReferenceResolver | None native_config: NativeModelConfigSession - - -@dataclass(frozen=True) -class SqlAnalysisOptOutRequest: - """One model's opt-out, with what is needed to prove its SQL parses.""" - - model_file: DiscoveredSqlModelFile - config: CompileModelConfig - settings: SettingsConfig - no_sql_validation: bool - query_sql: str - placeholders: dict[str, str] | None - project_config_path: Path diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index bc6f801e0b..0b9da83ea9 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -3,18 +3,583 @@ from __future__ import annotations from enum import StrEnum -from typing import TYPE_CHECKING, Protocol +from typing import TYPE_CHECKING, Any, Protocol from sqlbuild.sql_values.models import SqlValue if TYPE_CHECKING: + from collections.abc import Sequence + from pathlib import Path + + import sqlbuild._native as _native + from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds + from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.models import ( CompactBatchPreparation, - CompileProjectInputs, - SqlReferenceOrigin, - SqlReferenceScan, + CompiledLineageColumnFact, + CompiledObjectKey, + CompiledRelationLocation, + CompiledSeed, + CompiledSource, + CompiledSqlExpansion, + CompiledSqlScenario, + CompiledSqlTest, + CompileModelConfig, + CompilerDiagnostic, + CompileSeedInput, + CompileSourceInput, + CompileSqlReference, + CompileSqlScenarioInput, + CompileSqlTestInput, + DeclarationScopeBuild, + DynamicColumnContractProof, + FunctionArgument, + FunctionReturnColumn, + InferredColumn, + LoadedMacro, + MacroContext, + ) + from sqlbuild.compiler.discovery.models import ( + ConstantDeclaration, + DiscoveredAuditBlock, + DiscoveredAuditFile, + DiscoveredHookFunction, + DiscoveredLoaderFunction, + DiscoveredMaterializationFile, + DiscoveredProjectInputs, + DiscoveredPythonFunctionFile, + DiscoveredSchemaFile, + DiscoveredSeedFile, + DiscoveredSourceFile, + DiscoveredSqlFunctionFile, + DiscoveredSqlHookFile, + DiscoveredSqlModelFile, + EnumDeclaration, ) + from sqlbuild.compiler.references.models import LiteralSqlRelation + from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver + from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + from sqlbuild.lint.types import RelationKeyRow + from sqlbuild.spec.contracts.models import ( + LocalConfig, + ProjectConfig, + ScenarioConfig, + SchemaModelEntry, + SchemaSeedEntry, + SettingsConfig, + SourceEntry, + SourceLocation, + TargetConfig, + ) + + +class SourceInput(Protocol): + """One attached source: a native owner view, or a hand-built `CompileSourceInput`.""" + + @property + def source_entry(self) -> SourceEntry: ... + @property + def source_file(self) -> DiscoveredSourceFile: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def assembly_row(self) -> tuple[str, bool, str | None, str | None]: ... + + +class CompiledSourceView(Protocol): + """One assembled source: a native owner view, or a hand-built `CompiledSource`.""" + + @property + def key(self) -> CompiledObjectKey: ... + @property + def deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def source_entry(self) -> SourceEntry: ... + @property + def source_file(self) -> DiscoveredSourceFile: ... + + +class SeedInput(Protocol): + """One seed paired with its declaration: a native owner view, or a `CompileSeedInput`.""" + + @property + def seed_file(self) -> DiscoveredSeedFile: ... + @property + def schema_entry(self) -> SchemaSeedEntry: ... + @property + def schema_file(self) -> DiscoveredSchemaFile: ... + + +class CompiledSeedView(Protocol): + """One assembled seed: a native owner view, or a hand-built `CompiledSeed`.""" + + @property + def key(self) -> CompiledObjectKey: ... + @property + def deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def seed_file(self) -> DiscoveredSeedFile: ... + @property + def schema_entry(self) -> SchemaSeedEntry: ... + @property + def schema_file(self) -> DiscoveredSchemaFile: ... + @property + def destination(self) -> CompiledRelationLocation: ... + @property + def external(self) -> bool: ... + + +class FunctionInput(Protocol): + """One attached function: a native owner view, or a hand-built `CompileSqlFunctionInput`.""" + + @property + def function_file(self) -> DiscoveredSqlFunctionFile | DiscoveredPythonFunctionFile: ... + @property + def name(self) -> str: ... + @property + def arguments(self) -> tuple[FunctionArgument, ...]: ... + @property + def returns(self) -> str: ... + @property + def body_sql(self) -> str: ... + @property + def return_columns(self) -> tuple[FunctionReturnColumn, ...]: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def language(self) -> FunctionLanguage: ... + @property + def runtime_version(self) -> str | None: ... + @property + def entry_point(self) -> str | None: ... + @property + def packages(self) -> tuple[str, ...]: ... + @property + def tags(self) -> tuple[str, ...]: ... + @property + def description(self) -> str | None: ... + @property + def database(self) -> str | None: ... + @property + def schema(self) -> str | None: ... + @property + def logical_database(self) -> str | None: ... + @property + def logical_schema(self) -> str | None: ... + @property + def fingerprint_database(self) -> str | None: ... + @property + def fingerprint_schema(self) -> str | None: ... + @property + def fingerprint_logical_database(self) -> str | None: ... + @property + def fingerprint_logical_schema(self) -> str | None: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def assembly_row( + self, + ) -> tuple[str, str, list[str], list[tuple[str, str, str | None]]]: ... + + +class CompiledFunctionView(Protocol): + """One assembled function: a native owner view, or a hand-built `CompiledFunction`.""" + + @property + def key(self) -> CompiledObjectKey: ... + @property + def deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def arguments(self) -> tuple[FunctionArgument, ...]: ... + @property + def returns(self) -> str: ... + @property + def body_sql(self) -> str: ... + @property + def return_columns(self) -> tuple[FunctionReturnColumn, ...]: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def language(self) -> FunctionLanguage: ... + @property + def runtime_version(self) -> str | None: ... + @property + def entry_point(self) -> str | None: ... + @property + def packages(self) -> tuple[str, ...]: ... + @property + def tags(self) -> tuple[str, ...]: ... + @property + def description(self) -> str | None: ... + @property + def relative_path(self) -> Path: ... + @property + def source_file_path(self) -> Path | None: ... + @property + def destination(self) -> CompiledRelationLocation: ... + @property + def fingerprint_destination(self) -> CompiledRelationLocation: ... + + +class AuditInput(Protocol): + """One attached audit: a native owner view, or a hand-built `CompileAuditInput`.""" + + @property + def audit_file(self) -> DiscoveredAuditFile: ... + @property + def audit_block(self) -> DiscoveredAuditBlock: ... + @property + def sql_body(self) -> str: ... + @property + def evaluation_mode(self) -> AuditEvaluationMode: ... + @property + def measurement_contract(self) -> MeasurementContract | None: ... + @property + def thresholds(self) -> MeasurementThresholds | None: ... + @property + def minimum_samples(self) -> int | None: ... + @property + def measure_sql(self) -> str | None: ... + @property + def evidence_sql(self) -> str | None: ... + @property + def evidence_limit(self) -> int | None: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def attached_target_kind(self) -> AttachedAuditTargetKind | None: ... + @property + def attached_target_name(self) -> str | None: ... + @property + def attached_column_name(self) -> str | None: ... + @property + def severity(self) -> AuditSeverity | None: ... + @property + def run_scope(self) -> str | None: ... + @property + def always_run(self) -> bool: ... + @property + def description(self) -> str | None: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def name(self) -> str | None: ... + @property + def assembly_row( + self, + ) -> tuple[list[tuple[str, str, str | None]], tuple[str, str] | None]: ... + + +class CompiledAuditView(Protocol): + """One assembled audit: a native owner view, or a hand-built `CompiledAudit`.""" + + @property + def key(self) -> CompiledObjectKey: ... + @property + def scope_deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def definition_name(self) -> str: ... + @property + def audit_file(self) -> DiscoveredAuditFile: ... + @property + def audit_block(self) -> DiscoveredAuditBlock: ... + @property + def sql_body(self) -> str: ... + @property + def evaluation_mode(self) -> AuditEvaluationMode: ... + @property + def measurement_contract(self) -> MeasurementContract | None: ... + @property + def thresholds(self) -> MeasurementThresholds | None: ... + @property + def minimum_samples(self) -> int | None: ... + @property + def measure_sql(self) -> str | None: ... + @property + def evidence_sql(self) -> str | None: ... + @property + def evidence_limit(self) -> int | None: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def attached_target_kind(self) -> AttachedAuditTargetKind | None: ... + @property + def attached_target_name(self) -> str | None: ... + @property + def attached_column_name(self) -> str | None: ... + @property + def severity(self) -> AuditSeverity | None: ... + @property + def run_scope(self) -> str | None: ... + @property + def always_run(self) -> bool: ... + @property + def description(self) -> str | None: ... + + +class CompileInputsView(Protocol): + """A compile's attached inputs: a native owner view, or hand-built `CompileProjectInputs`.""" + + @property + def project_config(self) -> ProjectConfig: ... + @property + def local_config(self) -> LocalConfig: ... + @property + def discovered_inputs(self) -> DiscoveredProjectInputs: ... + @property + def sql_lexical_syntax(self) -> SqlLexicalSyntax: ... + @property + def run_id(self) -> str: ... + @property + def effective_target_name(self) -> str | None: ... + @property + def effective_target(self) -> TargetConfig | None: ... + @property + def compile_cache_dir(self) -> Path | None: ... + @property + def effective_connection(self) -> dict[str, object]: ... + @property + def effective_settings(self) -> SettingsConfig: ... + @property + def no_sql_validation(self) -> bool: ... + @property + def effective_vars(self) -> dict[str, object]: ... + @property + def macro_context(self) -> MacroContext | None: ... + @property + def loaded_macros(self) -> dict[str, LoadedMacro]: ... + @property + def public_enums(self) -> dict[str, EnumDeclaration]: ... + @property + def public_constants(self) -> dict[str, ConstantDeclaration]: ... + @property + def model_inputs(self) -> tuple[ModelInput, ...]: ... + @property + def seed_inputs(self) -> tuple[CompileSeedInput, ...]: ... + @property + def source_inputs(self) -> tuple[CompileSourceInput, ...]: ... + @property + def sql_function_inputs(self) -> tuple[FunctionInput, ...]: ... + @property + def test_inputs(self) -> tuple[CompileSqlTestInput, ...]: ... + @property + def scenario_inputs(self) -> tuple[CompileSqlScenarioInput, ...]: ... + @property + def audit_inputs(self) -> tuple[AuditInput, ...]: ... + @property + def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... + @property + def external_sql_reference_resolver(self) -> ExternalSqlReferenceResolver | None: ... + @property + def scope_index(self) -> ScopeIndex: ... + @property + def declaration_scope(self) -> DeclarationScopeBuild | None: ... + @property + def native_project(self) -> _native.NativeCompiledProject | None: ... + + +class CompiledProjectView(Protocol): + """One compiled project: a native owner view, or a hand-built `CompiledProject`.""" + + @property + def run_id(self) -> str: ... + @property + def effective_target_name(self) -> str | None: ... + @property + def effective_connection(self) -> dict[str, object]: ... + @property + def effective_vars(self) -> dict[str, object]: ... + @property + def binding_catalog(self) -> Any | None: ... + @property + def effective_target_database(self) -> str | None: ... + @property + def effective_target_schema(self) -> str | None: ... + @property + def sql_analysis_dialect(self) -> str | None: ... + @property + def sql_lexical_syntax(self) -> SqlLexicalSyntax: ... + @property + def compile_cache_dir(self) -> Path | None: ... + @property + def settings(self) -> SettingsConfig: ... + @property + def scenario(self) -> ScenarioConfig: ... + @property + def models(self) -> tuple[CompiledModelView, ...]: ... + @property + def sources(self) -> tuple[CompiledSource, ...]: ... + @property + def seeds(self) -> tuple[CompiledSeed, ...]: ... + @property + def functions(self) -> tuple[CompiledFunctionView, ...]: ... + @property + def audits(self) -> tuple[CompiledAuditView, ...]: ... + @property + def sql_tests(self) -> tuple[CompiledSqlTest, ...]: ... + @property + def sql_scenarios(self) -> tuple[CompiledSqlScenario, ...]: ... + @property + def loader_functions(self) -> tuple[DiscoveredLoaderFunction, ...]: ... + @property + def hook_functions(self) -> tuple[DiscoveredHookFunction, ...]: ... + @property + def enforce_explicit_references(self) -> bool: ... + @property + def sql_hook_files(self) -> tuple[DiscoveredSqlHookFile, ...]: ... + @property + def materialization_files(self) -> tuple[DiscoveredMaterializationFile, ...]: ... + @property + def public_enums(self) -> dict[str, EnumDeclaration]: ... + @property + def public_constants(self) -> dict[str, ConstantDeclaration]: ... + @property + def loaded_macros(self) -> dict[str, LoadedMacro]: ... + @property + def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... + @property + def unmatched_literal_sql_relations(self) -> tuple[LiteralSqlRelation, ...]: ... + @property + def external_sql_reference_resolver(self) -> ExternalSqlReferenceResolver | None: ... + @property + def scope_index(self) -> ScopeIndex: ... + @property + def sql_expansions(self) -> dict[Path, CompiledSqlExpansion]: ... + @property + def native_session(self) -> Any | None: ... + @property + def native_project(self) -> _native.NativeCompiledProject | None: ... + @property + def lineage_graph(self) -> _native.NativeProjectGraph: ... + + +class CompiledModelView(Protocol): + """One compiled model: a native owner view, or a hand-built `CompiledModel`.""" + + @property + def key(self) -> CompiledObjectKey: ... + @property + def deps(self) -> tuple[CompiledObjectKey, ...]: ... + @property + def name(self) -> str: ... + @property + def relative_path(self) -> Path: ... + @property + def query_sql(self) -> str: ... + @property + def config(self) -> CompileModelConfig: ... + @property + def destination(self) -> CompiledRelationLocation: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def schema_entry(self) -> SchemaModelEntry | None: ... + @property + def inferred_columns(self) -> tuple[InferredColumn, ...] | None: ... + @property + def fast_lineage_columns(self) -> Sequence[CompiledLineageColumnFact] | None: ... + @property + def fast_lineage_has_star(self) -> bool: ... + @property + def fast_lineage_star_resolved(self) -> bool: ... + @property + def authored_sql(self) -> str: ... + @property + def authored_query_sql(self) -> str: ... + @property + def output_column_locations(self) -> dict[str, SourceLocation]: ... + @property + def extract_implicit_alias_columns(self) -> bool: ... + @property + def macro_deps(self) -> tuple[str, ...]: ... + @property + def enum_declarations(self) -> tuple[EnumDeclaration, ...]: ... + @property + def constant_declarations(self) -> tuple[ConstantDeclaration, ...]: ... + @property + def enum_columns(self) -> dict[str, EnumDeclaration]: ... + @property + def binding_diagnostics(self) -> tuple[CompilerDiagnostic, ...]: ... + @property + def binding_validated(self) -> bool: ... + @property + def dynamic_column_contract(self) -> DynamicColumnContractProof | None: ... + @property + def unchecked_output_columns(self) -> frozenset[str]: ... + @property + def rejected_sql_analysis_opt_out(self) -> SourceLocation | None: ... + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... + @property + def relation_key_row(self) -> RelationKeyRow: ... + @property + def key_row(self) -> tuple[str, str]: ... + @property + def dep_rows(self) -> list[tuple[str, str]]: ... + @property + def destination_row(self) -> tuple[str | None, str | None, str]: ... + @property + def inferred_rows(self) -> list[tuple[str, str | None, str]] | None: ... + @property + def schema_column_rows(self) -> list[tuple[str, str | None]] | None: ... + @property + def has_dynamic_columns(self) -> bool: ... + @property + def output_column_count(self) -> int: ... + + +class ModelInput(Protocol): + """One model's compile input: a native owner view, or a hand-built `CompileModelInput`.""" + + @property + def model_file(self) -> DiscoveredSqlModelFile: ... + @property + def config(self) -> CompileModelConfig: ... + @property + def query_sql(self) -> str: ... + @property + def macro_source_sql(self) -> str: ... + @property + def references(self) -> tuple[CompileSqlReference, ...]: ... + @property + def schema_entry(self) -> SchemaModelEntry | None: ... + @property + def schema_file(self) -> DiscoveredSchemaFile | None: ... + @property + def sql_validation_enabled(self) -> bool: ... + @property + def rejected_sql_analysis_opt_out(self) -> SourceLocation | None: ... + @property + def enum_declarations(self) -> tuple[EnumDeclaration, ...]: ... + @property + def constant_declarations(self) -> tuple[ConstantDeclaration, ...]: ... + @property + def enum_columns(self) -> dict[str, EnumDeclaration]: ... + @property + def macro_deps(self) -> tuple[str, ...]: ... + @property + def macro_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def declaration_usages(self) -> tuple[UsageRecord, ...]: ... + @property + def sql_expansion(self) -> CompiledSqlExpansion | None: ... + @property + def lint_expansion(self) -> CompiledSqlExpansion | None: ... + @property + def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... + @property + def attached_schema_name(self) -> str | None: ... + @property + def relation_key_row(self) -> RelationKeyRow: ... class TypedSqlValueRenderer(Protocol): @@ -38,16 +603,10 @@ class CompactBatchResponseCallback(Protocol): def __call__(self, *, preparation: CompactBatchPreparation, response: object) -> None: ... -class SqlReferenceExtractor(Protocol): - """Keyword-only reference extraction for one expanded SQL text.""" - - def __call__(self, *, sql: str, origin: SqlReferenceOrigin | None) -> SqlReferenceScan: ... - - class CompileInputsReadyCallback(Protocol): """Invocation-local observer for the completed pre-semantic input snapshot.""" - def __call__(self, *, inputs: CompileProjectInputs) -> None: ... + def __call__(self, *, inputs: CompileInputsView) -> None: ... class AttachedAuditTargetKind(StrEnum): diff --git a/src/sqlbuild/compiler/compiled_project/__init__.py b/src/sqlbuild/compiler/compiled_project/__init__.py new file mode 100644 index 0000000000..31c048c1f1 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/__init__.py @@ -0,0 +1 @@ +"""The compile's natively owned project and the recorders producers retain facts with.""" diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/__init__.py b/src/sqlbuild/compiler/compiled_project/_helpers/__init__.py new file mode 100644 index 0000000000..d6aacb385a --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/_helpers/__init__.py @@ -0,0 +1 @@ +"""Compiled-project recording helpers.""" diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py b/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py new file mode 100644 index 0000000000..d15a8b542a --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py @@ -0,0 +1,214 @@ +"""SQL tests, scenarios and audits attached natively by the compile's model loop.""" + +from __future__ import annotations + +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.auditing.main._builtins import builtin_audit_definitions +from sqlbuild.compiler.auditing.types import AuditEvaluationMode +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache +from sqlbuild.compiler.compile.constants import SQL_TEST_CTE_SCAN_ALGORITHM +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import ( + CompileDirectLogicSqlTestInputPayload, + CompileModelSqlTestInputPayload, + CompilerDiagnostic, + CompileSqlScenarioCte, + CompileSqlScenarioInput, + CompileSqlTestCte, + CompileSqlTestInput, +) +from sqlbuild.compiler.compile.types import ( + AuditInput, + SourceInput, + SqlTestMode, + TypedSqlValueRenderer, +) +from sqlbuild.compiler.compiled_project._helpers import resource_locators +from sqlbuild.compiler.compiled_project._helpers.resource_rows import ( + loader_rows, + python_resource_rows, +) +from sqlbuild.compiler.discovery.models import ( + DiscoveredAuditFile, + DiscoveredProjectInputs, + DiscoveredSqlScenarioFile, + DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, +) +from sqlbuild.compiler.profiling.main.record import record_compile_timing +from sqlbuild.compiler.scopes.models import DeclarationIdentity +from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, UsageKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.spec.contracts.models import SettingsConfig, SourceLocation +from sqlbuild.sql_values.exceptions import SqlValueRenderingError, SqlValueValidationError +from sqlbuild.sql_values.models import SqlValue + + +def attach_sql_tests_impl( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: _native.NativeCompiledProject, + value_renderer: TypedSqlValueRenderer, + syntax: SqlLexicalSyntax, + scan_cache: SqlTestScanCache, +) -> tuple[tuple[CompileSqlTestInput, ...], tuple[CompileSqlScenarioInput, ...]]: + """Attach every SQL test case, then every scenario, saving the scan store afterwards.""" + + with record_compile_timing("test_input_compile_ms"): + tests, scenarios = native_project.attach_sql_tests( + [_test_file_row(test_file) for test_file in discovered_inputs.test_files], + [_scenario_row(scenario) for scenario in discovered_inputs.scenario_files], + { + "test_input": CompileSqlTestInput, + "model_payload": CompileModelSqlTestInputPayload, + "direct_payload": CompileDirectLogicSqlTestInputPayload, + "test_cte": CompileSqlTestCte, + "scenario_input": CompileSqlScenarioInput, + "scenario_cte": CompileSqlScenarioCte, + "modes": {mode.value: mode for mode in SqlTestMode}, + "test_kind": ResourceKind.TEST, + "scenario_kind": ResourceKind.SCENARIO, + "runtime_usage": UsageKind.RUNTIME, + "render_parameter": _parameter_renderer(value_renderer), + "adapter_name": value_renderer.adapter_name, + "source_location": SourceLocation, + "scan_store": scan_cache.opened(), + "scan_key": (SQL_TEST_CTE_SCAN_ALGORITHM, syntax.cache_key), + }, + ) + scan_cache.save() + return tuple(tests), tuple(scenarios) + + +def attach_audits_impl( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: _native.NativeCompiledProject, + effective_settings: SettingsConfig, +) -> tuple[tuple[AuditInput, ...], tuple[CompilerDiagnostic, ...]]: + """Attach every singular audit, then each model's, source's and seed's generic audits.""" + + default_severity: object | None = effective_settings.default_audit_severity + audits, diagnostics = native_project.attach_audits( + [_audit_file_row(audit_file) for audit_file in discovered_inputs.audit_files], + { + "default_severity": None if default_severity is None else str(default_severity), + "default_run_scope": effective_settings.default_audit_run_scope, + "builtins": [_audit_file_row(audit_file) for audit_file in builtin_audit_definitions()], + "declaration_identity": DeclarationIdentity, + "singular_kind": DeclarationKind.SINGULAR_AUDIT, + "audit_kind": DeclarationKind.AUDIT, + "resource_kinds": {kind.value: kind for kind in ResourceKind}, + }, + ) + return tuple(audits), tuple(diagnostics) + + +def _parameter_renderer(value_renderer: TypedSqlValueRenderer) -> object: + def render(value: SqlValue) -> tuple[str | None, str | None]: + try: + return value_renderer.render_typed_scalar(value=value), None + except (SqlValueRenderingError, SqlValueValidationError) as error: + return None, str(error) + + return render + + +def _test_file_row(test_file: DiscoveredSqlTestFile) -> tuple[object, ...]: + return ( + str(test_file.file_path), + str(test_file.relative_path), + test_file.relative_path.stem, + test_file, + test_file.contents, + [_test_block_row(block) for block in test_file.blocks], + ) + + +def _test_block_row(block: DiscoveredSqlTestBlock) -> tuple[object, ...]: + return ( + block, + block.name, + block.sql_body, + block.mode.value, + [parameter.name for parameter in block.parameters], + [(case, case.name, list(case.values)) for case in block.cases], + block.sql_body_span, + ) + + +def _scenario_row(scenario: DiscoveredSqlScenarioFile) -> tuple[object, ...]: + return ( + str(scenario.file_path), + str(scenario.relative_path), + scenario, + scenario.contents, + scenario.sql_body, + scenario.name, + ) + + +def _audit_file_row(audit_file: DiscoveredAuditFile) -> tuple[object, ...]: + return ( + str(audit_file.file_path), + str(audit_file.relative_path), + audit_file.file_path.stem, + audit_file, + audit_file.declaration_kind is DeclarationKind.AUDIT, + audit_file.contents, + [ + ( + block, + block.name, + block.sql_body, + block.evidence_sql, + block.evaluation_mode == AuditEvaluationMode.MEASUREMENT, + block.header_values, + ) + for block in audit_file.blocks + ], + ) + + +def resource_diagnostics_impl( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: _native.NativeCompiledProject, + source_inputs: tuple[SourceInput, ...], +) -> tuple[CompilerDiagnostic, ...]: + """Orphaned audit factory warnings, then missing descriptions, then cursor-input errors.""" + + orphans, descriptions, cursors = native_project.resource_diagnostics( + { + "compiler_diagnostic": CompilerDiagnostic, + "compile_error": CompileInputError, + "source_location": SourceLocation, + "path_class": Path, + "model_files": list(discovered_inputs.model_files), + "audit_factories": [ + (factory.name, str(factory.relative_path), factory.line) + for factory in discovered_inputs.audit_factories + ], + "scenarios": [ + ( + scenario.name, + str(scenario.relative_path), + scenario.contents, + scenario.header_values.get("description"), + ) + for scenario in discovered_inputs.scenario_files + ], + "sql_hooks": [ + (hook.name, str(hook.relative_path), hook.contents, hook.description) + for hook in discovered_inputs.sql_hook_files + ], + "loaders": loader_rows(discovered_inputs), + "python_resources": python_resource_rows( + discovered_inputs=discovered_inputs, source_inputs=source_inputs + ), + "locators": resource_locators, + } + ) + return (*orphans, *descriptions, *cursors) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py b/src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py new file mode 100644 index 0000000000..c107089052 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py @@ -0,0 +1,100 @@ +"""Project declarations and scope classes the native model loop resolves models with.""" + +import inspect +from collections.abc import Callable +from inspect import Parameter + +from sqlbuild.compiler.compile.models import DeclarationScopeResolver, ModelInputBuildContext +from sqlbuild.compiler.discovery.models import EnumDeclaration +from sqlbuild.compiler.scopes.models import ( + DeclarationIdentity, + DeclarationRecord, + ResourceIdentity, + VisibilityRecord, +) +from sqlbuild.compiler.scopes.types import ( + DeclarationKind, + ResourceKind, + ScopeKind, + VisibilityReason, +) + +_NAMED_KINDS: frozenset[DeclarationKind] = frozenset( + { + DeclarationKind.AUDIT, + DeclarationKind.SCHEMA, + DeclarationKind.SQL_HOOK, + DeclarationKind.PYTHON_HOOK, + } +) + + +def named_records( + resolver: DeclarationScopeResolver | None, +) -> dict[tuple[str, str], DeclarationRecord]: + """Each project-owned generic audit's, schema's and hook's first record, by `(kind, name)`.""" + + if resolver is None: + return {} + records: dict[tuple[str, str], DeclarationRecord] = {} + for identity, declarations in resolver.lookup.declarations.items(): + if identity.kind in _NAMED_KINDS and identity.owner is None and declarations: + records[identity.kind.value, identity.name] = declarations[0] + return records + + +def hook_signature(function: Callable[..., object]) -> list[tuple[str, int, bool]]: + """A hook function's parameters as `inspect.signature` reads them: name, kind, default.""" + + return [ + (parameter.name, int(parameter.kind), parameter.default is not Parameter.empty) + for parameter in inspect.signature(function).parameters.values() + ] + + +def schema_enums(resolver: DeclarationScopeResolver | None) -> dict[str, EnumDeclaration]: + """Project-owned enums by name, as `model_schema_enum_declarations` resolves column types.""" + + if resolver is None: + return {} + enums: dict[str, EnumDeclaration] = {} + for identity, value in resolver.projection.declarations.items(): + if ( + identity.kind is DeclarationKind.ENUM + and identity.owner is None + and isinstance(value, EnumDeclaration) + ): + enums[identity.name] = value + return enums + + +def visibility_classes( + *, context: ModelInputBuildContext, visible_declarations: object +) -> dict[str, object]: + """What the loop resolves each model's visible declarations from.""" + + resolver: DeclarationScopeResolver | None = context.declaration_resolver + return { + "resolver": resolver, + "projects": resolver is not None and bool(resolver.lookup.index.declarations), + "resource_specific": ( + frozenset( + declaration.identity.owner + for declaration in resolver.lookup.index.declarations + if declaration.scope is ScopeKind.PRIVATE and declaration.identity.owner is not None + ) + | frozenset(resolver.lookup.grants_by_resource) + if resolver is not None + else frozenset() + ), + "public_enums": context.public_enums, + "public_constants": context.public_constants, + "visible_declarations": visible_declarations, + "resource_identity": ResourceIdentity, + "declaration_identity": DeclarationIdentity, + "model_kind": ResourceKind.MODEL, + "enum_kind": DeclarationKind.ENUM, + "constant_kind": DeclarationKind.CONSTANT, + "private_owner": VisibilityReason.PRIVATE_OWNER, + "visibility_record": VisibilityRecord, + } diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py b/src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py new file mode 100644 index 0000000000..b346300292 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py @@ -0,0 +1,220 @@ +"""Record what later compile stages finish for a model, natively when its owner holds it.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import fields, replace +from typing import Any, cast + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import ( + CompiledAudit, + CompiledFunction, + CompiledModel, + CompiledProject, + CompiledSeed, + CompiledSource, + CompileProjectInputs, + CompilerDiagnostic, + InferredColumn, +) +from sqlbuild.compiler.compile.types import ( + CompiledAuditView, + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledSeedView, + CompiledSourceView, + CompileInputsView, +) + + +def compiled_model_record_impl(model: CompiledModelView) -> CompiledModel: + """The model as a `CompiledModel`, projected from its owner when it is a native view.""" + + if isinstance(model, CompiledModel): + return model + return CompiledModel( + **{field.name: getattr(model, field.name) for field in fields(CompiledModel)} + ) + + +def with_binding_diagnostics_impl( + *, model: CompiledModelView, diagnostics: tuple[CompilerDiagnostic, ...] +) -> CompiledModelView: + """The model with these final binding diagnostics.""" + + if isinstance(model, CompiledModel): + return replace(model, binding_diagnostics=diagnostics) + return cast(_native.NativeCompiledModel, model).with_binding_diagnostics(diagnostics) + + +def with_type_recovery_impl( + *, + model: CompiledModelView, + diagnostics: tuple[CompilerDiagnostic, ...], + poisoned: frozenset[tuple[str, str]], +) -> CompiledModelView: + """The model with its recovered bindings and its poisoned outputs' types forgotten.""" + + if not isinstance(model, CompiledModel): + native: _native.NativeCompiledModel = cast(_native.NativeCompiledModel, model) + return native.with_type_recovery( + diagnostics, [column for name, column in poisoned if name == native.name] + ) + columns: tuple[InferredColumn, ...] | None = None + if model.inferred_columns is not None: + columns = tuple( + replace(column, type=None) if (model.name, column.name) in poisoned else column + for column in model.inferred_columns + ) + unchecked: frozenset[str] = frozenset(column for name, column in poisoned if name == model.name) + return replace( + model, + binding_diagnostics=diagnostics, + inferred_columns=columns, + unchecked_output_columns=unchecked, + ) + + +def with_target_defaults_impl( + *, + model: _native.NativeCompiledModel, + defaults: tuple[str | None, str | None], + render_qualified_name: Callable[..., str | None], +) -> CompiledModelView: + """The model with the adapter's `(schema, database)` defaults filled into its destination.""" + + return model.with_target_defaults(defaults, render_qualified_name) + + +def compiled_project_record_impl( + *, project: CompiledProjectView, include_sql_expansions: bool = True +) -> CompiledProject: + """The project as a `CompiledProject`, projected from its owner when it is a native view.""" + + if isinstance(project, CompiledProject): + return project if include_sql_expansions else replace(project, sql_expansions={}) + fields: dict[str, Any] = cast(_native.NativeProjectView, project).fields(include_sql_expansions) + fields["seeds"] = tuple(map(compiled_seed_record_impl, fields["seeds"])) + fields["sources"] = tuple(map(compiled_source_record_impl, fields["sources"])) + fields["functions"] = tuple(map(compiled_function_record_impl, fields["functions"])) + fields["audits"] = tuple(map(compiled_audit_record_impl, fields["audits"])) + return CompiledProject(**fields) + + +def compiled_audit_record_impl(audit: CompiledAuditView) -> CompiledAudit: + """The audit as a `CompiledAudit`, projected from its owner when it is a native view.""" + + if isinstance(audit, CompiledAudit): + return audit + return CompiledAudit( + key=audit.key, + scope_deps=audit.scope_deps, + name=audit.name, + definition_name=audit.definition_name, + audit_file=audit.audit_file, + audit_block=audit.audit_block, + sql_body=audit.sql_body, + evaluation_mode=audit.evaluation_mode, + measurement_contract=audit.measurement_contract, + thresholds=audit.thresholds, + minimum_samples=audit.minimum_samples, + measure_sql=audit.measure_sql, + evidence_sql=audit.evidence_sql, + evidence_limit=audit.evidence_limit, + references=audit.references, + attached_target_kind=audit.attached_target_kind, + attached_target_name=audit.attached_target_name, + attached_column_name=audit.attached_column_name, + severity=audit.severity, + run_scope=audit.run_scope, + always_run=audit.always_run, + description=audit.description, + ) + + +def compiled_function_record_impl(function: CompiledFunctionView) -> CompiledFunction: + """The function as a `CompiledFunction`, projected from its owner when it is a native view.""" + + if isinstance(function, CompiledFunction): + return function + return CompiledFunction( + key=function.key, + deps=function.deps, + name=function.name, + relative_path=function.relative_path, + arguments=function.arguments, + returns=function.returns, + body_sql=function.body_sql, + destination=function.destination, + fingerprint_destination=function.fingerprint_destination, + return_columns=function.return_columns, + references=function.references, + language=function.language, + source_file_path=function.source_file_path, + runtime_version=function.runtime_version, + entry_point=function.entry_point, + packages=function.packages, + tags=function.tags, + description=function.description, + ) + + +def compiled_source_record_impl(source: CompiledSourceView) -> CompiledSource: + """The source as a `CompiledSource`, projected from its owner when it is a native view.""" + + if isinstance(source, CompiledSource): + return source + return CompiledSource( + key=source.key, + deps=source.deps, + name=source.name, + source_entry=source.source_entry, + source_file=source.source_file, + ) + + +def compiled_seed_record_impl(seed: CompiledSeedView) -> CompiledSeed: + """The seed as a `CompiledSeed`, projected from its owner when it is a native view.""" + + if isinstance(seed, CompiledSeed): + return seed + return CompiledSeed( + key=seed.key, + deps=seed.deps, + name=seed.name, + seed_file=seed.seed_file, + schema_entry=seed.schema_entry, + schema_file=seed.schema_file, + destination=seed.destination, + external=seed.external, + ) + + +def project_with_impl(*, project: CompiledProjectView, **changes: Any) -> CompiledProjectView: + """The project with `changes`; a native view shares its owner's models.""" + + if isinstance(project, CompiledProject): + return replace(project, **changes) + return cast(_native.NativeProjectView, project).with_changes(**changes) + + +def inputs_with_impl(*, inputs: CompileInputsView, **changes: Any) -> CompileInputsView: + """The compile inputs with `changes`; a native view shares its owner's model inputs.""" + + if isinstance(inputs, CompileProjectInputs): + return replace(inputs, **changes) + return cast(_native.NativeInputsView, inputs).with_changes(**changes) + + +def compile_inputs_view_impl( + *, fields: dict[str, Any], native_project: _native.NativeCompiledProject | None +) -> CompileInputsView: + """The native owner's inputs view when it holds every model input; else a record.""" + + if native_project is not None and all( + isinstance(model, _native.NativeModelInput) for model in fields["model_inputs"] + ): + return native_project.inputs_view(fields) + return CompileProjectInputs(**fields, native_project=native_project) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py b/src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py new file mode 100644 index 0000000000..0b27e7e4b1 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py @@ -0,0 +1,119 @@ +"""Seeds, functions and sources attached natively and held by the compile's native project.""" + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.types import FunctionInput, SeedInput, SourceInput +from sqlbuild.compiler.discovery.models import ( + DiscoveredProjectInputs, + DiscoveredSchemaFile, + DiscoveredSourceFile, +) +from sqlbuild.compiler.scopes.types import ResourceKind +from sqlbuild.spec.contracts.models import ( + DefaultsConfig, + SchemaSeedEntry, + SettingsConfig, + SourceEntry, + TargetConfig, +) + + +def attach_seeds_impl( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: _native.NativeCompiledProject, +) -> tuple[SeedInput, ...]: + """Pair every declared seed with its CSV file; raise for a declaration without one.""" + + declarations: list[tuple[SchemaSeedEntry, DiscoveredSchemaFile]] = [] + for schema_file in discovered_inputs.schema_files: + declarations.extend((seed_entry, schema_file) for seed_entry in schema_file.seed_entries) + return tuple( + native_project.attach_seeds( + declarations, list(discovered_inputs.seed_files), CompileInputError + ) + ) + + +def attach_functions_impl( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: _native.NativeCompiledProject, + settings: tuple[SettingsConfig, TargetConfig | None, str], + no_sql_validation: bool, + python_functions_inherit_default_namespace: bool, +) -> tuple[FunctionInput, ...]: + """Attach and validate every SQL, then Python, function in discovery order.""" + + effective_settings, target_config, adapter_name = settings + defaults: DefaultsConfig = discovered_inputs.project_config.defaults + sql_rows: list[tuple[object, ...]] = [ + ( + str(function_file.file_path), + str(function_file.relative_path), + function_file.file_path.stem, + function_file, + function_file.header_values, + function_file.body_sql, + function_file.contents, + ) + for function_file in discovered_inputs.sql_function_files + ] + python_rows: list[tuple[object, ...]] = [ + ( + str(function_file.file_path), + str(function_file.relative_path), + function_file.file_path.stem, + function_file, + function_file.header_values, + function_file.body_python, + None, + ) + for function_file in discovered_inputs.python_function_files + ] + return tuple( + native_project.attach_functions( + (sql_rows, python_rows), + ( + adapter_name, + effective_settings.sql_analysis and not no_sql_validation, + ( + defaults.function_database + if defaults.function_database is not None + else defaults.database, + defaults.function_schema + if defaults.function_schema is not None + else defaults.schema, + ), + None if target_config is None else (target_config.database, target_config.schema), + python_functions_inherit_default_namespace, + ResourceKind.FUNCTION, + ), + ) + ) + + +def attach_sources_impl( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: _native.NativeCompiledProject, + effective_settings: SettingsConfig, + no_sql_validation: bool, +) -> tuple[SourceInput, ...]: + """Expand and validate every discovered source in declaration order.""" + + rows: list[tuple[str, DiscoveredSourceFile, SourceEntry]] = [] + for source_file in discovered_inputs.source_files: + rows.extend( + (str(source_file.file_path), source_file, entry) for entry in source_file.source_entries + ) + return tuple( + native_project.attach_sources( + rows, + ( + discovered_inputs.project_config.defaults.contract, + effective_settings.sql_analysis and not no_sql_validation, + ResourceKind.SOURCE, + ), + ) + ) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py b/src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py new file mode 100644 index 0000000000..9fdee295d1 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/_helpers/resource_locators.py @@ -0,0 +1,29 @@ +"""The decorator locator the native resource diagnostics call for authored Python functions.""" + +from __future__ import annotations + +import ast + + +def decorator_location(*, contents: str, decorator: str) -> tuple[int, int]: + """The `@decorator` line and column offset, ignoring docstrings and comments.""" + + try: + module: ast.Module = ast.parse(contents) + except SyntaxError: + return 1, 1 + for node in ast.walk(module): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + for expression in node.decorator_list: + if _decorator_name(expression) == decorator: + return expression.lineno, expression.col_offset + return 1, 1 + + +def _decorator_name(expression: ast.expr) -> str | None: + target: ast.expr = expression.func if isinstance(expression, ast.Call) else expression + if isinstance(target, ast.Name): + return target.id + if isinstance(target, ast.Attribute): + return target.attr + return None diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/resource_rows.py b/src/sqlbuild/compiler/compiled_project/_helpers/resource_rows.py new file mode 100644 index 0000000000..d596ecbb38 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/_helpers/resource_rows.py @@ -0,0 +1,135 @@ +"""Rows the native resource diagnostics read: loaders and user Python resources.""" + +from __future__ import annotations + +import inspect +from collections.abc import Callable, Iterator +from functools import partial +from pathlib import Path + +from sqlbuild.compiler.compile.types import SourceInput +from sqlbuild.compiler.compiled_project.types import PythonResourceRow +from sqlbuild.compiler.discovery.constants import YAML_FILE_SUFFIXES +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider +from sqlbuild.errors.setting_help.main.snippet_help import snippet_help +from sqlbuild.spec.contracts.models import SourceLocation + + +def loader_rows(discovered_inputs: DiscoveredProjectInputs) -> dict[str, tuple[object, ...]]: + """Each loader's description, relative path and whether that path is a YAML file.""" + + return { + loader.name: ( + loader.description, + str(loader.relative_path), + loader.relative_path.suffix in YAML_FILE_SUFFIXES, + ) + for loader in discovered_inputs.loader_functions + } + + +def python_resource_rows( + *, discovered_inputs: DiscoveredProjectInputs, source_inputs: tuple[SourceInput, ...] +) -> list[PythonResourceRow]: + """User Python loaders, tasks, assets, checks and hooks, then providers, in report order.""" + + return [ + *_python_nodes(discovered_inputs=discovered_inputs, source_inputs=source_inputs), + *(_provider(provider) for provider in discovered_inputs.providers), + ] + + +def _python_nodes( + *, discovered_inputs: DiscoveredProjectInputs, source_inputs: tuple[SourceInput, ...] +) -> Iterator[PythonResourceRow]: + source_owned_loaders: frozenset[str] = frozenset( + source_input.source_entry.loader + for source_input in source_inputs + if source_input.source_entry.loader is not None + ) + source_paths: frozenset[Path] = frozenset( + source_file.relative_path for source_file in discovered_inputs.source_files + ) + for loader in discovered_inputs.loader_functions: + if loader.name in source_owned_loaders or loader.relative_path in source_paths: + continue + yield _python_resource( + kind="loader", + name=loader.name, + description=loader.description, + relative_path=loader.relative_path, + function=loader.function, + ) + for kind, functions in ( + ("task", discovered_inputs.task_functions), + ("asset", discovered_inputs.asset_functions), + ("check", discovered_inputs.check_functions), + ("hook", discovered_inputs.hook_functions), + ): + for function in functions: + yield _python_resource( + kind=kind, + name=function.name, + description=function.description, + relative_path=function.relative_path, + function=function.function, + ) + + +def _python_resource( + *, + kind: str, + name: str, + description: str | None, + relative_path: Path, + function: Callable[..., object], +) -> PythonResourceRow: + return ( + kind, + name, + description, + partial(_function_location, path=relative_path, function=function), + snippet_help( + purpose=f"to describe {kind} '{name}'", + target=( + f"give its function in {relative_path.as_posix()} a docstring " + f"or pass a description to @{kind}" + ), + lines=(f'@{kind}(description="What {name} does", ...)',), + ), + ) + + +def _provider(provider: DiscoveredProvider) -> PythonResourceRow: + class_name: str = provider.provider_class.__name__ + raw_doc: object | None = provider.provider_class.__dict__.get("__doc__") + return ( + "provider", + provider.name, + inspect.cleandoc(raw_doc) if isinstance(raw_doc, str) else None, + partial( + _class_location, path=provider.relative_path, provider_class=provider.provider_class + ), + snippet_help( + purpose=f"to describe provider '{provider.name}'", + target=f"add a class docstring in {provider.relative_path.as_posix()}", + lines=( + f"class {class_name}(Provider):", + f' """What {provider.name} provides."""', + ), + ), + ) + + +def _function_location(*, path: Path, function: Callable[..., object]) -> SourceLocation: + code: object | None = getattr(inspect.unwrap(function), "__code__", None) + line: object | None = getattr(code, "co_firstlineno", None) + return SourceLocation(path=path, line=line if isinstance(line, int) else 1, column=1) + + +def _class_location(*, path: Path, provider_class: type[object]) -> SourceLocation: + try: + line: int = inspect.getsourcelines(provider_class)[1] + except (OSError, TypeError): + line = 1 + return SourceLocation(path=path, line=line, column=1) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/rows.py b/src/sqlbuild/compiler/compiled_project/_helpers/rows.py new file mode 100644 index 0000000000..8e069f74f7 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/_helpers/rows.py @@ -0,0 +1,196 @@ +"""Rows producers hand the native compiled project when they retain a resource's facts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import ( + CompileModelConfig, + CompileSqlReference, + DynamicColumnContractProof, +) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView +from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration +from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.spec.contracts.models import SchemaModelEntry +from sqlbuild.sql_values.models import SqlValue +from sqlbuild.sql_values.types import SqlValueKind + + +@dataclass(frozen=True) +class _ModelDeclarations: + enum_columns: tuple[str, ...] + enums: tuple[EnumDeclaration, ...] + constants: tuple[ConstantDeclaration, ...] + + +def record_model_analyses_impl( + *, project: _native.NativeCompiledProject, models: tuple[CompiledModelView, ...] +) -> int: + """Transfer each assembled model's typed analysis to its retained facts; return how many.""" + + transferred: int = project.record_view_analyses( + [model for model in models if isinstance(model, _native.NativeCompiledModel)] + ) + for model in models: + if isinstance(model, _native.NativeCompiledModel): + continue + proof: DynamicColumnContractProof | None = model.dynamic_column_contract + inferred: list[tuple[str, str | None, str]] | None = model.inferred_rows + columns: list[tuple[str, str | None, str]] = inferred or [] + transferred += project.record_model_analysis( + model.name, + ( + inferred is not None, + [(name, column_type) for name, column_type, _ in columns], + [nullability == InferredNullability.NULLABLE for _, _, nullability in columns], + bool(model.fast_lineage_has_star), + list(model.unchecked_output_columns), + ), + None + if proof is None + else ( + proof.output_proven, + proof.bare_dynamic_pivot, + [(family.name.casefold(), family.inferred_type) for family in proof.families], + proof.failure_reason, + [(family.name, family.inferred_type) for family in proof.families], + ), + ) + return transferred + + +def hand_built_project_impl(project: CompiledProjectView) -> _native.NativeCompiledProject: + """A native project for a `CompiledProject` built without the compile-input stage.""" + + native: _native.NativeCompiledProject = _native.NativeCompiledProject() + for model in project.models: + native.record_model( + _model_row( + identity=( + model.name, + model.relative_path.as_posix(), + model.query_sql, + model.authored_query_sql, + model.authored_sql, + ), + config=model.config, + references=model.references, + schema_entry=model.schema_entry, + relative_path=model.relative_path, + declarations=_ModelDeclarations( + enum_columns=tuple(model.enum_columns), + enums=model.enum_declarations, + constants=model.constant_declarations, + ), + ) + ) + _ = record_model_analyses_impl(project=native, models=project.models) + return native + + +def enum_row(declaration: EnumDeclaration) -> tuple[object, ...]: + """An enum declaration as the native project row.""" + + return ( + declaration.name, + declaration.relative_path.as_posix(), + [(member.name, member.value) for member in declaration.members], + None, + None, + None, + ) + + +def constant_row(declaration: ConstantDeclaration) -> tuple[object, ...]: + """A constant declaration as the native project row, its value made JSON-safe.""" + + return ( + declaration.name, + declaration.relative_path.as_posix(), + [], + typed_value_payload(declaration.value), + declaration.logical_type.display_name, + declaration.render_as.value if declaration.render_as is not None else None, + ) + + +def typed_value_payload(value: SqlValue) -> object: + """A typed SQL value as the plain value native consumers read.""" + + if value.kind == SqlValueKind.DECIMAL: + return str(cast(Decimal, value.value)) + if value.kind in { + SqlValueKind.STRING, + SqlValueKind.INTEGER, + SqlValueKind.BOOLEAN, + SqlValueKind.FLOAT, + SqlValueKind.NULL, + }: + return value.value + if value.kind in {SqlValueKind.LIST, SqlValueKind.SET}: + return [typed_value_payload(item) for item in cast(tuple[SqlValue, ...], value.value)] + return { + key: typed_value_payload(item) + for key, item in cast(tuple[tuple[str, SqlValue], ...], value.value) + } + + +def _model_row( + *, + identity: tuple[str, str, str, str, str], + config: CompileModelConfig, + references: tuple[CompileSqlReference, ...], + schema_entry: SchemaModelEntry | None, + relative_path: Path, + declarations: _ModelDeclarations, +) -> tuple[object, ...]: + return ( + identity, + (config.values, list(config.model_header_keys), config.layer_schema), + [ + (str(reference.ref_kind), reference.ref_name, reference.ref_package) + for reference in references + ], + _schema_row(entry=schema_entry, relative_path=relative_path), + list(declarations.enum_columns), + ( + [enum_row(declaration) for declaration in declarations.enums], + [constant_row(declaration) for declaration in declarations.constants], + ), + ) + + +def _schema_row(*, entry: SchemaModelEntry | None, relative_path: Path) -> object: + if entry is None: + return None + return ( + len(entry.audits), + [ + ( + column.name, + column.type, + column.nullable, + len(column.audits), + column.location is not None and column.location.path != relative_path, + ) + for column in entry.columns + ], + [ + ( + family.name.casefold(), + family.name, + family.pivot_column, + family.value_column, + family.aggregate, + family.type, + family.name_pattern, + ) + for family in entry.dynamic_columns + ], + (bool(entry.type_enforcement), entry.model_schema is not None), + ) diff --git a/src/sqlbuild/compiler/compiled_project/constants.py b/src/sqlbuild/compiler/compiled_project/constants.py new file mode 100644 index 0000000000..5208508531 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/constants.py @@ -0,0 +1,9 @@ +"""Constants of the native compiled project and its model loop.""" + +from sqlbuild.compiler.planner.types import MaterializationType + +TABLE_BACKED_MATERIALIZATIONS: tuple[MaterializationType, ...] = ( + MaterializationType.TABLE, + MaterializationType.INCREMENTAL, + MaterializationType.SNAPSHOT, +) diff --git a/src/sqlbuild/compiler/compiled_project/main/__init__.py b/src/sqlbuild/compiler/compiled_project/main/__init__.py new file mode 100644 index 0000000000..5de9ede1eb --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/__init__.py @@ -0,0 +1 @@ +"""Compiled-project entrypoints.""" diff --git a/src/sqlbuild/compiler/compiled_project/main/_attach_checks.py b/src/sqlbuild/compiler/compiled_project/main/_attach_checks.py new file mode 100644 index 0000000000..80c703c218 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/_attach_checks.py @@ -0,0 +1,61 @@ +"""SQL tests, scenarios and audits attached natively and held by the compile's native project.""" + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache +from sqlbuild.compiler.compile.models import ( + CompilerDiagnostic, + CompileSqlScenarioInput, + CompileSqlTestInput, +) +from sqlbuild.compiler.compile.types import AuditInput, SourceInput, TypedSqlValueRenderer +from sqlbuild.compiler.compiled_project._helpers.assurance_attachment import ( + attach_audits_impl, + attach_sql_tests_impl, + resource_diagnostics_impl, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.spec.contracts.models import SettingsConfig + + +def attach_checks( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: _native.NativeCompiledProject, + settings: tuple[SettingsConfig, TypedSqlValueRenderer, SqlLexicalSyntax], + scan_cache: SqlTestScanCache, + source_inputs: tuple[SourceInput, ...], +) -> tuple[ + tuple[CompileSqlTestInput, ...], + tuple[CompileSqlScenarioInput, ...], + tuple[AuditInput, ...], + tuple[CompilerDiagnostic, ...], +]: + """Attach every SQL test case, scenario and audit, then report every resource diagnostic.""" + + effective_settings, value_renderer, syntax = settings + tests, scenarios = attach_sql_tests_impl( + discovered_inputs=discovered_inputs, + native_project=native_project, + value_renderer=value_renderer, + syntax=syntax, + scan_cache=scan_cache, + ) + audits, diagnostics = attach_audits_impl( + discovered_inputs=discovered_inputs, + native_project=native_project, + effective_settings=effective_settings, + ) + return ( + tests, + scenarios, + audits, + ( + *diagnostics, + *resource_diagnostics_impl( + discovered_inputs=discovered_inputs, + native_project=native_project, + source_inputs=source_inputs, + ), + ), + ) diff --git a/src/sqlbuild/compiler/compiled_project/main/_attach_resources.py b/src/sqlbuild/compiler/compiled_project/main/_attach_resources.py new file mode 100644 index 0000000000..02e9d66513 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/_attach_resources.py @@ -0,0 +1,40 @@ +"""Seeds, functions and sources attached natively and held by the compile's native project.""" + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import FunctionInput, SeedInput, SourceInput +from sqlbuild.compiler.compiled_project._helpers.resource_attachment import ( + attach_functions_impl, + attach_seeds_impl, + attach_sources_impl, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.spec.contracts.models import SettingsConfig, TargetConfig + + +def attach_resources( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: _native.NativeCompiledProject, + settings: tuple[SettingsConfig, TargetConfig | None, str], + no_sql_validation: bool, + python_functions_inherit_default_namespace: bool, +) -> tuple[tuple[SeedInput, ...], tuple[FunctionInput, ...], tuple[SourceInput, ...]]: + """Pair every seed, then attach every function and source, in that order.""" + + seeds: tuple[SeedInput, ...] = attach_seeds_impl( + discovered_inputs=discovered_inputs, native_project=native_project + ) + functions: tuple[FunctionInput, ...] = attach_functions_impl( + discovered_inputs=discovered_inputs, + native_project=native_project, + settings=settings, + no_sql_validation=no_sql_validation, + python_functions_inherit_default_namespace=python_functions_inherit_default_namespace, + ) + sources: tuple[SourceInput, ...] = attach_sources_impl( + discovered_inputs=discovered_inputs, + native_project=native_project, + effective_settings=settings[0], + no_sql_validation=no_sql_validation, + ) + return seeds, functions, sources diff --git a/src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py b/src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py new file mode 100644 index 0000000000..1b3cba941f --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py @@ -0,0 +1,15 @@ +"""A compile's attached inputs, held by its native owner when it finished every model.""" + +from typing import Any + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompileInputsView +from sqlbuild.compiler.compiled_project._helpers.model_updates import compile_inputs_view_impl + + +def compile_inputs_view( + *, fields: dict[str, Any], native_project: _native.NativeCompiledProject | None +) -> CompileInputsView: + """The native owner's inputs view when it holds every model input; else a record.""" + + return compile_inputs_view_impl(fields=fields, native_project=native_project) diff --git a/src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py b/src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py new file mode 100644 index 0000000000..48c83f1684 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py @@ -0,0 +1,135 @@ +"""The Python classes and project inputs the native model loop builds models with.""" + +from dataclasses import replace +from functools import partial +from pathlib import Path +from typing import cast + +from sqlbuild.compiler.auditing.types import AuditSeverity +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import ( + CompactLineageFacts, + CompiledLineageColumnFact, + CompiledLineageSourceFact, + CompiledObjectKey, + CompiledRelationLocation, + CompiledSqlExpansion, + CompilerDiagnostic, + CompileSqlReference, + DeclarationScopeResolver, + DynamicColumnContractProof, + DynamicColumnFamilyProof, + ExpansionSpan, + FunctionArgument, + FunctionReturnColumn, + InferredColumn, + ModelInputBuildContext, + NativeModelConfigSession, +) +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, FunctionLanguage +from sqlbuild.compiler.compiled_project._helpers.loop_inputs import ( + hook_signature, + named_records, + schema_enums, + visibility_classes, +) +from sqlbuild.compiler.compiled_project.main._native_config_classes import native_config_classes +from sqlbuild.compiler.compiled_project.main.declaration_rows import declaration_rows +from sqlbuild.compiler.discovery.models import ( + DiscoveredAuditFactory, + DiscoveredHookFunction, + PythonHookEntry, + SqlHookEntry, +) +from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.compiler.model_config.main._native_config_error import native_config_error +from sqlbuild.compiler.planner.types import ContractPolicy +from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind +from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError +from sqlbuild.compiler.scopes.models import UsageRecord +from sqlbuild.compiler.scopes.types import UsageKind +from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation + + +def model_loop_classes( + *, + audit_factories: tuple[DiscoveredAuditFactory, ...], + context: ModelInputBuildContext, + external_resolver: ExternalSqlReferenceResolver | None, + host_inputs: dict[str, object], +) -> dict[str, object]: + """Return the classes, error factories and project declarations the model loop reads.""" + + resolver: DeclarationScopeResolver | None = context.declaration_resolver + hook_functions: tuple[DiscoveredHookFunction, ...] = cast( + tuple[DiscoveredHookFunction, ...], host_inputs["hook_functions"] + ) + provider_names: frozenset[str] = cast(frozenset[str], host_inputs["provider_names"]) + session: NativeModelConfigSession = cast( + NativeModelConfigSession, host_inputs["config_session"] + ) + return { + "usage_record": UsageRecord, + "compile_error": CompileInputError, + "syntax_help": host_inputs["syntax_help"], + "source_syntax_help": host_inputs["source_syntax_help"], + "macros": host_inputs["macros"], + "sql_hooks": host_inputs["sql_hooks"], + "scope_path_error": InvalidScopePathError, + "sql_hook_entry": SqlHookEntry, + "config_error": native_config_error, + "audit_factories": {factory.name: factory for factory in audit_factories}, + "generated_usage": UsageKind.GENERATED, + "contract_enforced": ContractPolicy.ENFORCED, + "model_schemas": context.public_model_schemas, + "named_records": named_records(resolver), + "schema_enums": schema_enums(resolver), + "compiler_diagnostic": CompilerDiagnostic, + "report_diagnostic": host_inputs["report_diagnostic"], + "path_class": Path, + "external_resolver": external_resolver, + "reference_kind": SqlReferenceKind, + "partial": partial, + "config": native_config_classes( + session=session, record_template_reads=host_inputs["record_template_reads"] + ), + "hooks": { + "functions": {function.name: function for function in hook_functions}, + "provider_names": sorted(provider_names), + "python_hook_entry": PythonHookEntry, + "signature": hook_signature, + }, + "visibility": visibility_classes( + context=context, visible_declarations=host_inputs["visible_declarations"] + ), + "record": {"replace": replace, "declaration_rows": declaration_rows}, + "input_view": { + "sql_reference": CompileSqlReference, + "reference_kind": SqlReferenceKind, + "schema_model_entry": SchemaModelEntry, + "sql_expansion": CompiledSqlExpansion, + "expansion_span": ExpansionSpan, + "audits": { + "audit_severity": AuditSeverity, + "attached_target_kind": AttachedAuditTargetKind, + }, + "functions": { + "function_argument": FunctionArgument, + "function_return_column": FunctionReturnColumn, + "function_language": FunctionLanguage, + }, + "compiled": { + "object_key": CompiledObjectKey, + "relation_location": CompiledRelationLocation, + "inferred_column": InferredColumn, + "nullability": InferredNullability, + "compiler_diagnostic": CompilerDiagnostic, + "source_location": SourceLocation, + "compact_lineage_facts": CompactLineageFacts, + "lineage_column_fact": CompiledLineageColumnFact, + "lineage_source_fact": CompiledLineageSourceFact, + "contract_proof": DynamicColumnContractProof, + "family_proof": DynamicColumnFamilyProof, + }, + }, + } diff --git a/src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py b/src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py new file mode 100644 index 0000000000..630cfcc029 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py @@ -0,0 +1,27 @@ +"""The builder, storage defaults and classes a model's `CompileModelConfig` is made of.""" + +from sqlbuild.compiler.compile.models import CompileModelConfig, NativeModelConfigSession +from sqlbuild.compiler.compiled_project.constants import TABLE_BACKED_MATERIALIZATIONS +from sqlbuild.spec.contracts.models import ResolvedTableType, ResolvedTimeTravelRetention +from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TimeTravelRetentionSource + + +def native_config_classes( + *, session: NativeModelConfigSession, record_template_reads: object +) -> dict[str, object]: + """The builder, storage defaults and classes a model's `CompileModelConfig` is made of.""" + + return { + "builder": session.builder, + "inherited_storage": session.inherited_storage, + "table_backed": tuple( + str(materialization) for materialization in TABLE_BACKED_MATERIALIZATIONS + ), + "compile_model_config": CompileModelConfig, + "resolved_retention": ResolvedTimeTravelRetention, + "retention_source": TimeTravelRetentionSource.MODEL, + "resolved_table_type": ResolvedTableType, + "table_type": TableType, + "table_type_source": TableTypeSource.MODEL, + "record_template_reads": record_template_reads, + } diff --git a/src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py b/src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py new file mode 100644 index 0000000000..437f3562ab --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py @@ -0,0 +1,13 @@ +"""Transfer assembled models' typed analysis results to the native compiled project.""" + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompiledModelView +from sqlbuild.compiler.compiled_project._helpers.rows import record_model_analyses_impl + + +def record_model_analyses( + *, project: _native.NativeCompiledProject, models: tuple[CompiledModelView, ...] +) -> int: + """Attach each model's inferred columns and dynamic pivot proof to its retained facts.""" + + return record_model_analyses_impl(project=project, models=models) diff --git a/src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py b/src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py new file mode 100644 index 0000000000..481393f0a7 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py @@ -0,0 +1,13 @@ +"""Record a compiled model's final binding diagnostics.""" + +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledModelView +from sqlbuild.compiler.compiled_project._helpers.model_updates import with_binding_diagnostics_impl + + +def with_binding_diagnostics( + *, model: CompiledModelView, diagnostics: tuple[CompilerDiagnostic, ...] +) -> CompiledModelView: + """The model with these final binding diagnostics.""" + + return with_binding_diagnostics_impl(model=model, diagnostics=diagnostics) diff --git a/src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py b/src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py new file mode 100644 index 0000000000..b449833d34 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py @@ -0,0 +1,20 @@ +"""Fill an adapter's default schema and database into a native compiled model's destination.""" + +from collections.abc import Callable + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompiledModelView +from sqlbuild.compiler.compiled_project._helpers.model_updates import with_target_defaults_impl + + +def with_target_defaults( + *, + model: _native.NativeCompiledModel, + defaults: tuple[str | None, str | None], + render_qualified_name: Callable[..., str | None], +) -> CompiledModelView: + """The model with the adapter's `(schema, database)` defaults filled into its destination.""" + + return with_target_defaults_impl( + model=model, defaults=defaults, render_qualified_name=render_qualified_name + ) diff --git a/src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py b/src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py new file mode 100644 index 0000000000..860c502cc7 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py @@ -0,0 +1,16 @@ +"""Record a compiled model's semantic type recovery.""" + +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledModelView +from sqlbuild.compiler.compiled_project._helpers.model_updates import with_type_recovery_impl + + +def with_type_recovery( + *, + model: CompiledModelView, + diagnostics: tuple[CompilerDiagnostic, ...], + poisoned: frozenset[tuple[str, str]], +) -> CompiledModelView: + """The model with its recovered bindings and its poisoned outputs' types forgotten.""" + + return with_type_recovery_impl(model=model, diagnostics=diagnostics, poisoned=poisoned) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py new file mode 100644 index 0000000000..bd551710db --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py @@ -0,0 +1,11 @@ +"""A compiled audit as a `CompiledAudit` record, for callers that copy or expose it.""" + +from sqlbuild.compiler.compile.models import CompiledAudit +from sqlbuild.compiler.compile.types import CompiledAuditView +from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_audit_record_impl + + +def compiled_audit_record(audit: CompiledAuditView) -> CompiledAudit: + """The audit as a `CompiledAudit`, projected from its owner when it is a native view.""" + + return compiled_audit_record_impl(audit) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py new file mode 100644 index 0000000000..c988babf6e --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py @@ -0,0 +1,12 @@ +"""A compiled function as a `CompiledFunction` record, for callers that copy or expose it.""" + +from sqlbuild.compiler.compile.types import CompiledFunctionView +from sqlbuild.compiler.compiled_project._helpers.model_updates import ( + compiled_function_record_impl, +) + + +def compiled_function_record(function: CompiledFunctionView) -> CompiledFunctionView: + """The function as a `CompiledFunction`, projected from its owner when it is a native view.""" + + return compiled_function_record_impl(function) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py new file mode 100644 index 0000000000..bcdc2613bb --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py @@ -0,0 +1,11 @@ +"""A compiled model as a `CompiledModel` record, for callers that copy or ship it.""" + +from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView +from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_model_record_impl + + +def compiled_model_record(model: CompiledModelView) -> CompiledModel: + """The model as a `CompiledModel`, projected from its owner when it is a native view.""" + + return compiled_model_record_impl(model) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py b/src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py new file mode 100644 index 0000000000..1c1b65fcca --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py @@ -0,0 +1,13 @@ +"""The native compiled project a consumer reads for one `CompiledProject`.""" + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project._helpers.rows import hand_built_project_impl + + +def compiled_project_facts(project: CompiledProjectView) -> _native.NativeCompiledProject: + """The compile's retained project, or one recorded from a hand-built project.""" + + if project.native_project is not None: + return project.native_project + return hand_built_project_impl(project) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py new file mode 100644 index 0000000000..d698d9c082 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py @@ -0,0 +1,15 @@ +"""A compiled project as a `CompiledProject` record, for callers that copy or ship it.""" + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_project_record_impl + + +def compiled_project_record( + project: CompiledProjectView, *, include_sql_expansions: bool = True +) -> CompiledProject: + """The project as a `CompiledProject`, projected from its owner when it is a native view.""" + + return compiled_project_record_impl( + project=project, include_sql_expansions=include_sql_expansions + ) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py new file mode 100644 index 0000000000..33f6724975 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py @@ -0,0 +1,11 @@ +"""A compiled seed as a `CompiledSeed` record, for callers that copy or expose it.""" + +from sqlbuild.compiler.compile.models import CompiledSeed +from sqlbuild.compiler.compile.types import CompiledSeedView +from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_seed_record_impl + + +def compiled_seed_record(seed: CompiledSeedView) -> CompiledSeed: + """The seed as a `CompiledSeed`, projected from its owner when it is a native view.""" + + return compiled_seed_record_impl(seed) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py new file mode 100644 index 0000000000..51c0c910e3 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py @@ -0,0 +1,11 @@ +"""A compiled source as a `CompiledSource` record, for callers that copy or expose it.""" + +from sqlbuild.compiler.compile.models import CompiledSource +from sqlbuild.compiler.compile.types import CompiledSourceView +from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_source_record_impl + + +def compiled_source_record(source: CompiledSourceView) -> CompiledSource: + """The source as a `CompiledSource`, projected from its owner when it is a native view.""" + + return compiled_source_record_impl(source) diff --git a/src/sqlbuild/compiler/compiled_project/main/declaration_rows.py b/src/sqlbuild/compiler/compiled_project/main/declaration_rows.py new file mode 100644 index 0000000000..d1c346b4a1 --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/declaration_rows.py @@ -0,0 +1,14 @@ +"""Declaration rows native consumers read.""" + +from sqlbuild.compiler.compiled_project._helpers.rows import constant_row, enum_row +from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration + + +def declaration_rows( + *, enums: tuple[EnumDeclaration, ...], constants: tuple[ConstantDeclaration, ...] +) -> tuple[list[tuple[object, ...]], list[tuple[object, ...]]]: + """Return enum and constant declarations as native rows, in the given order.""" + + return [enum_row(declaration) for declaration in enums], [ + constant_row(declaration) for declaration in constants + ] diff --git a/src/sqlbuild/compiler/compiled_project/main/inputs_with.py b/src/sqlbuild/compiler/compiled_project/main/inputs_with.py new file mode 100644 index 0000000000..f05e7123af --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/inputs_with.py @@ -0,0 +1,12 @@ +"""Compile inputs with some fields changed, natively when their owner holds them.""" + +from typing import Any + +from sqlbuild.compiler.compile.types import CompileInputsView +from sqlbuild.compiler.compiled_project._helpers.model_updates import inputs_with_impl + + +def inputs_with(*, inputs: CompileInputsView, **changes: Any) -> CompileInputsView: + """The compile inputs with `changes`; a native view shares its owner's model inputs.""" + + return inputs_with_impl(inputs=inputs, **changes) diff --git a/src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py b/src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py new file mode 100644 index 0000000000..2f3fe093dc --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py @@ -0,0 +1,10 @@ +"""Typed SQL values as the plain values native consumers read.""" + +from sqlbuild.compiler.compiled_project._helpers.rows import typed_value_payload +from sqlbuild.sql_values.models import SqlValue + + +def plain_typed_value(value: SqlValue) -> object: + """Return the value JSON-safe: decimals as text, collections and mappings recursively.""" + + return typed_value_payload(value) diff --git a/src/sqlbuild/compiler/compiled_project/main/project_with.py b/src/sqlbuild/compiler/compiled_project/main/project_with.py new file mode 100644 index 0000000000..bb011f16cc --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/main/project_with.py @@ -0,0 +1,12 @@ +"""A compiled project with some fields changed, natively when its owner holds it.""" + +from typing import Any + +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project._helpers.model_updates import project_with_impl + + +def project_with(*, project: CompiledProjectView, **changes: Any) -> CompiledProjectView: + """The project with `changes`; a native view shares its owner's models.""" + + return project_with_impl(project=project, **changes) diff --git a/src/sqlbuild/compiler/compiled_project/types.py b/src/sqlbuild/compiler/compiled_project/types.py new file mode 100644 index 0000000000..68f523dc8a --- /dev/null +++ b/src/sqlbuild/compiler/compiled_project/types.py @@ -0,0 +1,9 @@ +"""Row shapes the native model loop hands its Python host.""" + +from collections.abc import Callable + +from sqlbuild.spec.contracts.models import SourceLocation + +type ReferenceRow = tuple[str, str, str | None, int | None] +type SpanRow = tuple[int, int, int, int] +type PythonResourceRow = tuple[str, str, str | None, Callable[[], SourceLocation], str] diff --git a/src/sqlbuild/compiler/contract_adoption/_helpers/compare.py b/src/sqlbuild/compiler/contract_adoption/_helpers/compare.py index 56fc1845ec..d18dcedc51 100644 --- a/src/sqlbuild/compiler/contract_adoption/_helpers/compare.py +++ b/src/sqlbuild/compiler/contract_adoption/_helpers/compare.py @@ -7,15 +7,18 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.compile.models import CompiledModel, CompiledSource -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledResourceType, + CompiledSourceView, +) from sqlbuild.compiler.contract_adoption.models import ContractEvidence, ContractFinding from sqlbuild.compiler.contract_adoption.types import ContractFindingKind from sqlbuild.spec.contracts.main.matching_dynamic_families import matching_dynamic_families from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily, SourceEntry -def model_evidence(*, model: CompiledModel) -> ContractEvidence: +def model_evidence(*, model: CompiledModelView) -> ContractEvidence: return ContractEvidence( resource_type=CompiledResourceType.MODEL, resource_name=model.name, @@ -34,7 +37,7 @@ def model_evidence(*, model: CompiledModel) -> ContractEvidence: ) -def source_evidence(*, source: CompiledSource) -> ContractEvidence: +def source_evidence(*, source: CompiledSourceView) -> ContractEvidence: entry: SourceEntry = source.source_entry relation: str | None = entry.table or entry.name findings: tuple[ContractFinding, ...] = () diff --git a/src/sqlbuild/compiler/contract_adoption/_helpers/repository_edits.py b/src/sqlbuild/compiler/contract_adoption/_helpers/repository_edits.py index 6052335d1b..0a9440cc5d 100644 --- a/src/sqlbuild/compiler/contract_adoption/_helpers/repository_edits.py +++ b/src/sqlbuild/compiler/contract_adoption/_helpers/repository_edits.py @@ -1,486 +1,68 @@ -"""Metadata-preserving MODEL and source declaration edits.""" +"""Metadata-preserving MODEL and source declaration edits, computed natively.""" from __future__ import annotations -import re - +import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledModel, CompiledSource +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledSourceView from sqlbuild.compiler.contract_adoption.exceptions import ContractAdoptionError from sqlbuild.compiler.contract_adoption.models import ContractFinding from sqlbuild.compiler.contract_adoption.types import ContractFindingKind -from sqlbuild.compiler.discovery.main._model_header_spans import ( - get_model_header_spans, -) -from sqlbuild.compiler.discovery.models import ModelHeaderColumnSpan, ModelHeaderSpans -from sqlbuild.spec.contracts.main.matching_dynamic_families import matching_dynamic_families -from sqlbuild.spec.contracts.models import SchemaColumn, SourceColumnEntry - -_HEADER_WHITESPACE: str = " \t" -_GENERATED_MARKER: str = "generated" -_YAML_COLUMNS_KEY: str = "columns" -_YAML_TYPE_KEY: str = "type" -_YAML_KEY_SEPARATOR: str = ":" -_MODEL_HEADER_ENTRY_SEPARATOR: str = "," -_MODEL_HEADER_OPEN_PAREN: str = "(" -_MODEL_HEADER_CLOSE_PAREN: str = ")" -_MODEL_HEADER_OPEN_NESTING_CHARACTERS: frozenset[str] = frozenset({"(", "[", "{"}) -_MODEL_HEADER_CLOSE_NESTING_CHARACTERS: frozenset[str] = frozenset({")", "]", "}"}) -_MODEL_HEADER_ESCAPE_CHARACTER: str = "\\" -_MODEL_HEADER_WORD_EXTRA_CHARACTERS: frozenset[str] = frozenset({"_", "$"}) -_SINGLE_QUOTE: str = "'" -_DOUBLE_QUOTE: str = '"' -_MINIMUM_QUOTED_LENGTH: int = 2 -_UNQUOTED_MODEL_TYPE_PATTERN: re.Pattern[str] = re.compile( - r"^[A-Za-z_][A-Za-z0-9_$]*" - r"(?:\s*\(\s*[A-Za-z0-9_$+-]+(?:\s*,\s*[A-Za-z0-9_$+-]+)*\s*\))?$" -) +from sqlbuild.spec.contracts.models import SchemaModelEntry def edit_model( - *, model: CompiledModel, physical_columns: tuple[ColumnInfo, ...], overwrite: bool + *, model: CompiledModelView, physical_columns: tuple[ColumnInfo, ...], overwrite: bool ) -> tuple[str | None, str | None]: - if model.schema_entry is not None and model.schema_entry.model_schema is not None: - return None, "model uses a shared SCHEMA declaration" - if model.schema_entry is not None and any( - column.location is not None and column.location.path != model.relative_path - for column in model.schema_entry.columns - ): - return None, "effective columns are owned by an external schema file" - if model.schema_entry is not None and model.schema_entry.dynamic_columns: - fixed_names: frozenset[str] = frozenset( - column.name.casefold() for column in model.schema_entry.columns - ) - physical_columns = tuple( - column - for column in physical_columns - if column.name.casefold() in fixed_names - or not matching_dynamic_families( - families=model.schema_entry.dynamic_columns, - column_name=column.name, + schema: SchemaModelEntry | None = model.schema_entry + return _edit_outcome( + _native.contract_model_edit( + ( + model.authored_sql, + str(model.relative_path), + schema is not None and schema.model_schema is not None, + [ + ( + column.name, + column.type, + None if column.location is None else str(column.location.path), + ) + for column in (schema.columns if schema is not None else ()) + ], + [family.name_pattern for family in (schema.dynamic_columns if schema else ())], + [(column.name, column.type) for column in physical_columns], + overwrite, ) ) - contents: str = model.authored_sql - header_spans: ModelHeaderSpans = get_model_header_spans(contents=contents) - span_result: tuple[int, int, dict[str, ModelHeaderColumnSpan]] | None = header_spans.columns - if span_result is None: - header_span: tuple[int, int] | None = header_spans.body - if header_span is None: - return None, "MODEL header could not be edited safely" - insertion: int = header_span[1] - columns_sql: str = _render_model_columns(physical_columns) - prefix: str = "\n " - return contents[:insertion] + prefix + columns_sql + contents[insertion:], None - body_start, body_end, column_spans = span_result - declared_by_name: dict[str, SchemaColumn] = { - column.name.casefold(): column - for column in (model.schema_entry.columns if model.schema_entry is not None else ()) - } - physical_by_name: dict[str, ColumnInfo] = { - column.name.casefold(): column for column in physical_columns - } - edits: list[tuple[int, int, str]] = [] - for name, declared in declared_by_name.items(): - column_span: ModelHeaderColumnSpan | None = column_spans.get(declared.name) - if column_span is None: - return None, f"column '{declared.name}' has no unambiguous authored span" - physical: ColumnInfo | None = physical_by_name.get(name) - if physical is None: - if overwrite: - edits.append((column_span.entry_start, column_span.entry_end, "")) - continue - if declared.type is None: - metadata: str = contents[column_span.metadata_start : column_span.metadata_end] - separator: str = ", " if metadata.strip() else "" - edits.append( - ( - column_span.metadata_start, - column_span.metadata_start, - f"type {_render_model_type(physical.type)}{separator}", - ) - ) - elif overwrite and declared.type.casefold() != physical.type.casefold(): - replacement_span: tuple[int, int] | None = _declared_type_span( - contents=contents, - column_span=column_span, - declared_type=declared.type, - ) - if replacement_span is None: - return None, f"column '{declared.name}' type span could not be edited safely" - edits.append((*replacement_span, _render_model_type(physical.type))) - additions: tuple[ColumnInfo, ...] = tuple( - column for column in physical_columns if column.name.casefold() not in declared_by_name ) - if additions: - indentation: str = _column_indentation(contents=contents, body_start=body_start) - rendered: str = "".join( - f"\n{indentation}{column.name} (type {_render_model_type(column.type)})" - for column in additions - ) - edits.append((body_end, body_end, rendered + "\n ")) - return _apply_text_edits(contents=contents, edits=edits), None - - -def _render_model_columns(columns: tuple[ColumnInfo, ...]) -> str: - body: str = "".join( - f"\n {column.name} (type {_render_model_type(column.type)})" for column in columns - ) - return f"columns ({body}\n )" - - -def _render_model_type(value: str) -> str: - if _UNQUOTED_MODEL_TYPE_PATTERN.fullmatch(value): - return value - escaped: str = value.replace("\\", "\\\\").replace('"', '\\"') - return f'"{escaped}"' - - -def _column_indentation(*, contents: str, body_start: int) -> str: - next_line: int = contents.find("\n", body_start) - if next_line < 0: - return " " - index: int = next_line + 1 - while index < len(contents) and contents[index] in _HEADER_WHITESPACE: - index += 1 - return contents[next_line + 1 : index] or " " - - -def _declared_type_span( - *, contents: str, column_span: ModelHeaderColumnSpan, declared_type: str -) -> tuple[int, int] | None: - index: int = column_span.metadata_start - nesting_depth: int = 0 - while index < column_span.metadata_end: - character: str = contents[index] - if character in {_SINGLE_QUOTE, _DOUBLE_QUOTE}: - index = _quoted_value_end(contents=contents, start=index, end=column_span.metadata_end) - continue - if character in _MODEL_HEADER_OPEN_NESTING_CHARACTERS: - nesting_depth += 1 - index += 1 - continue - if character in _MODEL_HEADER_CLOSE_NESTING_CHARACTERS: - nesting_depth = max(0, nesting_depth - 1) - index += 1 - continue - if nesting_depth > 0: - index += 1 - continue - if not contents[index : index + len(_YAML_TYPE_KEY)].casefold() == _YAML_TYPE_KEY: - index += 1 - continue - key_end: int = index + len(_YAML_TYPE_KEY) - if ( - index > column_span.metadata_start and _is_header_word_character(contents[index - 1]) - ) or (key_end < column_span.metadata_end and _is_header_word_character(contents[key_end])): - index = key_end - continue - value_start: int = key_end - while value_start < column_span.metadata_end and contents[value_start].isspace(): - value_start += 1 - value_end: int - if contents[value_start : value_start + 1] in {_SINGLE_QUOTE, _DOUBLE_QUOTE}: - value_end = _quoted_value_end( - contents=contents, - start=value_start, - end=column_span.metadata_end, - ) - authored_value: str = contents[value_start + 1 : value_end - 1] - authored_value = authored_value.replace( - f"\\{contents[value_start]}", contents[value_start] - ).replace("\\\\", "\\") - else: - value_end = _unquoted_type_value_end( - contents=contents, - start=value_start, - end=column_span.metadata_end, - ) - authored_value = contents[value_start:value_end] - if _model_types_match(authored=authored_value, parsed=declared_type): - return value_start, value_end - index = value_end - return None - - -def _model_types_match(*, authored: str, parsed: str) -> bool: - if authored.casefold() == parsed.casefold(): - return True - if not ( - _UNQUOTED_MODEL_TYPE_PATTERN.fullmatch(authored) - and _UNQUOTED_MODEL_TYPE_PATTERN.fullmatch(parsed) - ): - return False - return re.sub(r"\s+", "", authored).casefold() == re.sub(r"\s+", "", parsed).casefold() - - -def _unquoted_type_value_end(*, contents: str, start: int, end: int) -> int: - index: int = start - parenthesis_depth: int = 0 - saw_open_parenthesis: bool = False - while index < end: - character: str = contents[index] - if character == _MODEL_HEADER_OPEN_PAREN: - parenthesis_depth += 1 - saw_open_parenthesis = True - elif character == _MODEL_HEADER_CLOSE_PAREN: - if parenthesis_depth == 0: - break - parenthesis_depth -= 1 - elif parenthesis_depth == 0 and character.isspace(): - next_non_whitespace: int = index - while next_non_whitespace < end and contents[next_non_whitespace].isspace(): - next_non_whitespace += 1 - if not saw_open_parenthesis and contents[ - next_non_whitespace : next_non_whitespace + 1 - ] == (_MODEL_HEADER_OPEN_PAREN): - index = next_non_whitespace - continue - break - elif parenthesis_depth == 0 and character == _MODEL_HEADER_ENTRY_SEPARATOR: - break - index += 1 - return index - - -def _quoted_value_end(*, contents: str, start: int, end: int) -> int: - quote: str = contents[start] - index: int = start + 1 - while index < end: - if contents[index] == _MODEL_HEADER_ESCAPE_CHARACTER: - index += 2 - continue - if contents[index] == quote: - return index + 1 - index += 1 - return end - - -def _is_header_word_character(character: str) -> bool: - return character.isalnum() or character in _MODEL_HEADER_WORD_EXTRA_CHARACTERS def edit_source( *, - source: CompiledSource, + source: CompiledSourceView, physical_columns: tuple[ColumnInfo, ...], overwrite: bool, contents: str | None = None, ) -> tuple[str | None, str | None]: - contents = source.source_file.contents if contents is None else contents - if _GENERATED_MARKER in "\n".join(source.source_file.contents.splitlines()[:5]).casefold(): - return None, "source file is marked as generated" - lines: list[str] = contents.splitlines(keepends=True) - source_range: tuple[int, int, int] | None = _yaml_named_block( - lines=lines, name=source.name, minimum_indent=0 - ) - if source_range is None: - return None, "source declaration has no unambiguous YAML block" - source_start, source_end, source_indent = source_range - columns_index: int | None = next( - ( - index - for index in range(source_start + 1, source_end) - if _yaml_key(lines[index]) == _YAML_COLUMNS_KEY - and _indent(lines[index]) == source_indent + 2 - ), - None, - ) - declared_by_name: dict[str, SourceColumnEntry] = { - column.name.casefold(): column for column in source.source_entry.columns - } - physical_by_name: dict[str, ColumnInfo] = { - column.name.casefold(): column for column in physical_columns - } - if columns_index is None: - insertion_lines: list[str] = [" " * (source_indent + 2) + "columns:\n"] - for column in physical_columns: - insertion_lines.extend(_yaml_column_lines(column=column, indent=source_indent + 4)) - lines[source_end:source_end] = insertion_lines - return "".join(lines), None - columns_indent: int = _indent(lines[columns_index]) - detected_column_indent: int | None = _yaml_sequence_indent( - lines=lines, - start=columns_index + 1, - end=source_end, - key_indent=columns_indent, - ) - column_indent: int = ( - detected_column_indent if detected_column_indent is not None else columns_indent + 2 - ) - columns_end: int = columns_index + 1 - while columns_end < source_end: - stripped: str = lines[columns_end].strip() - if stripped and not stripped.startswith("#"): - line_indent: int = _indent(lines[columns_end]) - if line_indent < columns_indent or ( - line_indent == columns_indent - and not ( - column_indent == columns_indent - and _yaml_list_name(lines[columns_end]) is not None - ) - ): - break - columns_end += 1 - column_blocks: dict[str, tuple[int, int]] = {} - index: int = columns_index + 1 - while index < columns_end: - parsed_name: str | None = _yaml_list_name(lines[index]) - if parsed_name is not None and _indent(lines[index]) == column_indent: - block_end: int = index + 1 - while block_end < columns_end and not ( - _indent(lines[block_end]) <= column_indent - and _yaml_list_name(lines[block_end]) is not None - ): - block_end += 1 - column_blocks[parsed_name.casefold()] = (index, block_end) - index = block_end - continue - index += 1 - line_edits: list[tuple[int, int, list[str]]] = [] - for name, declared in declared_by_name.items(): - block: tuple[int, int] | None = column_blocks.get(name) - if block is None: - return None, f"source column '{declared.name}' has no unambiguous YAML block" - physical: ColumnInfo | None = physical_by_name.get(name) - if physical is None: - if overwrite: - line_edits.append((block[0], block[1], [])) - continue - type_line: int | None = next( - (i for i in range(block[0] + 1, block[1]) if _yaml_key(lines[i]) == _YAML_TYPE_KEY), - None, - ) - if type_line is None: - line_edits.append( - ( - block[0] + 1, - block[0] + 1, - [" " * (column_indent + 2) + f"type: {_yaml_string(physical.type)}\n"], - ) - ) - elif ( - overwrite - and declared.type is not None - and declared.type.casefold() != physical.type.casefold() - ): - newline: str = "\n" if lines[type_line].endswith("\n") else "" - comment: str = _yaml_comment(lines[type_line]) - line_edits.append( - ( - type_line, - type_line + 1, - [ - " " * (column_indent + 2) - + f"type: {_yaml_string(physical.type)}" - + comment - + newline - ], - ) + return _edit_outcome( + _native.contract_source_edit( + ( + source.source_file.contents if contents is None else contents, + source.source_file.contents, + source.name, + [(column.name, column.type) for column in source.source_entry.columns], + [(column.name, column.type) for column in physical_columns], + overwrite, ) - additions: tuple[ColumnInfo, ...] = tuple( - column for column in physical_columns if column.name.casefold() not in declared_by_name + ) ) - if additions: - insertion: list[str] = [] - for column in additions: - insertion.extend(_yaml_column_lines(column=column, indent=column_indent)) - line_edits.append((columns_end, columns_end, insertion)) - for start, end, replacement_lines in sorted(line_edits, reverse=True): - lines[start:end] = replacement_lines - return "".join(lines), None - - -def _yaml_named_block( - *, lines: list[str], name: str, minimum_indent: int -) -> tuple[int, int, int] | None: - matches: list[tuple[int, int]] = [ - (index, _indent(line)) - for index, line in enumerate(lines) - if _yaml_list_name(line) == name and _indent(line) >= minimum_indent - ] - if not matches: - return None - minimum: int = min(indent for _, indent in matches) - candidates: list[int] = [index for index, indent in matches if indent == minimum] - if len(candidates) != 1: - return None - start: int = candidates[0] - end: int = start + 1 - while end < len(lines): - if _indent(lines[end]) <= minimum and _yaml_list_name(lines[end]) is not None: - break - end += 1 - return start, end, minimum - - -def _yaml_list_name(line: str) -> str | None: - stripped: str = line.strip() - if not stripped.startswith("- name:"): - return None - return _unquote(stripped[len("- name:") :].strip()) - - -def _yaml_key(line: str) -> str | None: - stripped: str = line.strip() - if not stripped or stripped.startswith(("#", "-")) or _YAML_KEY_SEPARATOR not in stripped: - return None - return stripped.split(_YAML_KEY_SEPARATOR, maxsplit=1)[0].strip() - - -def _yaml_sequence_indent(*, lines: list[str], start: int, end: int, key_indent: int) -> int | None: - for index in range(start, end): - stripped: str = lines[index].strip() - if not stripped or stripped.startswith("#"): - continue - line_indent: int = _indent(lines[index]) - if _yaml_list_name(lines[index]) is not None and line_indent >= key_indent: - return line_indent - if line_indent <= key_indent: - return None - return None - - -def _yaml_comment(line: str) -> str: - marker: int = line.find(" #") - if marker < 0: - return "" - return line[marker:].rstrip("\n") - - -def _yaml_string(value: str) -> str: - return "'" + value.replace("'", "''") + "'" - - -def _unquote(value: str) -> str: - if ( - len(value) >= _MINIMUM_QUOTED_LENGTH - and value[0] == value[-1] - and value[0] in {_SINGLE_QUOTE, _DOUBLE_QUOTE} - ): - return value[1:-1] - return value - - -def _indent(line: str) -> int: - return len(line) - len(line.lstrip(" ")) - - -def _yaml_column_lines(*, column: ColumnInfo, indent: int) -> list[str]: - return [ - " " * indent + f"- name: {column.name}\n", - " " * (indent + 2) + f"type: {_yaml_string(column.type)}\n", - ] -def _apply_text_edits(*, contents: str, edits: list[tuple[int, int, str]]) -> str: - result: str = contents - previous_start: int = len(contents) + 1 - for start, end, replacement_text in sorted(edits, reverse=True): - if end > previous_start: - raise ContractAdoptionError("overlapping contract source edits") - result = result[:start] + replacement_text + result[end:] - previous_start = start - return result +def _edit_outcome(row: tuple[str | None, str | None, str | None]) -> tuple[str | None, str | None]: + contents, conflict, failure = row + if failure is not None: + raise ContractAdoptionError(failure) + return contents, conflict def remaining_findings( diff --git a/src/sqlbuild/compiler/contract_adoption/main/compare.py b/src/sqlbuild/compiler/contract_adoption/main/compare.py index 1ff2e42ea6..43035953cd 100644 --- a/src/sqlbuild/compiler/contract_adoption/main/compare.py +++ b/src/sqlbuild/compiler/contract_adoption/main/compare.py @@ -4,7 +4,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.compiler.compile.models import CompiledModel, CompiledSource +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledSourceView from sqlbuild.compiler.contract_adoption._helpers.compare import ( compare_evidence, model_evidence, @@ -18,8 +18,8 @@ def compare_contracts( *, adapter: BaseAdapter, connection: object, - models: tuple[CompiledModel, ...], - sources: tuple[CompiledSource, ...], + models: tuple[CompiledModelView, ...], + sources: tuple[CompiledSourceView, ...], ) -> tuple[ContractEvidence, ...]: """Compare selected declarations with one bulk read of physical columns.""" diff --git a/src/sqlbuild/compiler/contract_adoption/main/write.py b/src/sqlbuild/compiler/contract_adoption/main/write.py index a8fb9975d2..426aee7550 100644 --- a/src/sqlbuild/compiler/contract_adoption/main/write.py +++ b/src/sqlbuild/compiler/contract_adoption/main/write.py @@ -8,10 +8,12 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, - CompiledModel, - CompiledSource, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledResourceType, + CompiledSourceView, +) from sqlbuild.compiler.contract_adoption._helpers.repository_edits import ( edit_model, edit_source, @@ -41,8 +43,10 @@ def write_contracts( ) -> ContractAdoptionResult: """Apply safe source-only edits atomically and validate the resulting project.""" - models: dict[str, CompiledModel] = {model.name: model for model in graph.project.models} - sources: dict[str, CompiledSource] = {source.name: source for source in graph.project.sources} + models: dict[str, CompiledModelView] = {model.name: model for model in graph.project.models} + sources: dict[str, CompiledSourceView] = { + source.name: source for source in graph.project.sources + } updates: dict[Path, str] = {} updated_evidence: list[ContractEvidence] = [] for evidence in result.evidence: diff --git a/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py b/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py index b296483fde..46aa691589 100644 --- a/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py +++ b/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py @@ -5,115 +5,51 @@ import sqlbuild._native as _native from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, CompilerDiagnostic, - DynamicColumnContractProof, RelatedLocation, ) from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, CompiledResourceType, DiagnosticPhase, DiagnosticSeverity, ) +from sqlbuild.compiler.compiled_project.main.compiled_project_facts import compiled_project_facts from sqlbuild.compiler.contracts._helpers.output_locations import ( output_column_location, output_related_locations, ) from sqlbuild.compiler.contracts.models import ContractValidationResult -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry, SourceLocation +from sqlbuild.spec.contracts.models import SourceLocation from sqlbuild.spec.contracts.types import ColumnContractMode -type _DeclaredRow = tuple[str, str | None, bool, bool] -type _SchemaRow = tuple[list[_DeclaredRow], list[tuple[str, str]], bool, bool] -type _ProofRow = tuple[bool, str | None, list[tuple[str, str | None]]] -type _ModelRow = tuple[ - str, - str | None, - _SchemaRow | None, - list[tuple[str, str | None, bool]] | None, - bool, - _ProofRow | None, - list[str], -] type _DiagnosticRow = tuple[ str, bool, str, str | None, int | None, str | None, tuple[str, str] | None, str ] def native_model_contracts( - *, project: CompiledProject, dialect: TypeDialect | str | None + *, project: CompiledProjectView, dialect: TypeDialect | str | None ) -> ContractValidationResult: - """Return every model's column contract diagnostics, in model order.""" + """Return every model's column contract diagnostics, read from the owner, in model order.""" mode: ColumnContractMode = project.settings.column_contract_mode - outcomes: list[list[_DiagnosticRow]] = _native.evaluate_native_model_contracts( - ( - str(dialect or "generic"), - mode == ColumnContractMode.IMPLICIT, - [_model_row(model) for model in project.models], + outcomes: dict[str, list[_DiagnosticRow]] = dict( + _native.evaluate_project_model_contracts( + (str(dialect or "generic"), mode == ColumnContractMode.IMPLICIT), + compiled_project_facts(project), ) ) diagnostics: list[CompilerDiagnostic] = [] - for model, rows in zip(project.models, outcomes, strict=True): - diagnostics.extend(_diagnostic(model=model, row=row) for row in rows) + for model in project.models: + diagnostics.extend( + _diagnostic(model=model, row=row) for row in outcomes.get(model.name, ()) + ) return ContractValidationResult(diagnostics=tuple(diagnostics)) -def _model_row(model: CompiledModel) -> _ModelRow: - contract: object = model.config.values.get("contract") - return ( - model.name, - contract if isinstance(contract, str) else None, - _schema_row(model=model, schema=model.schema_entry), - None - if model.inferred_columns is None - else [ - (column.name, column.type, column.nullability == InferredNullability.NULLABLE) - for column in model.inferred_columns - ], - bool(model.fast_lineage_has_star), - _proof_row(model.dynamic_column_contract), - list(model.unchecked_output_columns), - ) - - -def _schema_row(*, model: CompiledModel, schema: SchemaModelEntry | None) -> _SchemaRow | None: - if schema is None: - return None - named_schema: bool = schema.model_schema is not None - return ( - [ - ( - column.name, - column.type, - column.nullable is False, - named_schema and _declared_outside_model(model=model, column=column), - ) - for column in schema.columns - ], - [(family.name, family.type) for family in schema.dynamic_columns], - bool(schema.type_enforcement), - named_schema, - ) - - -def _declared_outside_model(*, model: CompiledModel, column: SchemaColumn) -> bool: - return column.location is not None and column.location.path != model.relative_path - - -def _proof_row(proof: DynamicColumnContractProof | None) -> _ProofRow | None: - if proof is None: - return None - return ( - bool(proof.output_proven), - proof.failure_reason, - [(family.name, family.inferred_type) for family in proof.families], - ) - - -def _diagnostic(*, model: CompiledModel, row: _DiagnosticRow) -> CompilerDiagnostic: +def _diagnostic(*, model: CompiledModelView, row: _DiagnosticRow) -> CompilerDiagnostic: code, is_error, message, column_name, declared_index, output_column, related, help_text = row return CompilerDiagnostic( phase=DiagnosticPhase.CONTRACT, @@ -131,7 +67,7 @@ def _diagnostic(*, model: CompiledModel, row: _DiagnosticRow) -> CompilerDiagnos def _location( - *, model: CompiledModel, declared_index: int | None, output_column: str | None + *, model: CompiledModelView, declared_index: int | None, output_column: str | None ) -> SourceLocation | None: if declared_index is not None and model.schema_entry is not None: return model.schema_entry.columns[declared_index].location @@ -141,7 +77,7 @@ def _location( def _related_locations( - *, model: CompiledModel, related: tuple[str, str] | None + *, model: CompiledModelView, related: tuple[str, str] | None ) -> tuple[RelatedLocation, ...]: if related is None: return () diff --git a/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py b/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py index c143956477..45cbab049f 100644 --- a/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py +++ b/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py @@ -2,30 +2,28 @@ from __future__ import annotations +from pathlib import Path + import sqlbuild._native as _native from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import ( + CompiledProjectView, CompiledResourceType, DiagnosticPhase, DiagnosticSeverity, ) -from sqlbuild.compiler.planner.types import MaterializationType +from sqlbuild.compiler.compiled_project.main.compiled_project_facts import compiled_project_facts def native_promotion_conflicts( - *, project: CompiledProject, adapter_default: TablePromotionMode, settings_file: str + *, project: CompiledProjectView, adapter_default: TablePromotionMode, settings_file: str ) -> tuple[CompilerDiagnostic, ...]: """Return one K011 error per enforced-contract table model under immediate promotion.""" explicit_mode: str | None = project.settings.table_promotion_mode - conflicts: list[tuple[int, str, str, str]] = _native.native_promotion_conflicts( - ( - explicit_mode, - str(adapter_default), - settings_file, - [_model_row(model) for model in project.models], - ) + conflicts: list[tuple[str, str, str, str, str]] = _native.project_promotion_conflicts( + (explicit_mode, str(adapter_default), settings_file), compiled_project_facts(project) ) return tuple( CompilerDiagnostic( @@ -34,23 +32,9 @@ def native_promotion_conflicts( code=code, message=message, resource_type=CompiledResourceType.MODEL, - resource_name=project.models[index].name, - path=project.models[index].relative_path, + resource_name=name, + path=Path(relative_path), help=help_text, ) - for index, code, message, help_text in conflicts + for name, relative_path, code, message, help_text in conflicts ) - - -def _model_row(model: CompiledModel) -> tuple[str, str | None, str | None, str | None]: - values: dict[str, object] = model.config.values - return ( - model.name, - _text(values.get("contract")), - _text(values.get("materialized", MaterializationType.TABLE)), - _text(values.get("incremental_mode")), - ) - - -def _text(value: object) -> str | None: - return str.__str__(value) if isinstance(value, str) else None diff --git a/src/sqlbuild/compiler/contracts/_helpers/output_locations.py b/src/sqlbuild/compiler/contracts/_helpers/output_locations.py index 70cf20d658..9fff6434b0 100644 --- a/src/sqlbuild/compiler/contracts/_helpers/output_locations.py +++ b/src/sqlbuild/compiler/contracts/_helpers/output_locations.py @@ -5,7 +5,8 @@ from functools import lru_cache from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledModel, RelatedLocation +from sqlbuild.compiler.compile.models import RelatedLocation +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.main._model_output_column_locations import ( extract_model_output_column_locations, ) @@ -13,7 +14,7 @@ def output_related_locations( - *, model: CompiledModel, column_name: str, message: str + *, model: CompiledModelView, column_name: str, message: str ) -> tuple[RelatedLocation, ...]: """The output column's location as a related location, when the model has one.""" @@ -23,7 +24,7 @@ def output_related_locations( return (RelatedLocation(label="output", location=location, message=message),) -def output_column_location(*, model: CompiledModel, column_name: str) -> SourceLocation | None: +def output_column_location(*, model: CompiledModelView, column_name: str) -> SourceLocation | None: """Where the model's SQL produces one output column.""" location: SourceLocation | None = model.output_column_locations.get(column_name) diff --git a/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py b/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py index e9e0e4489f..0e78152845 100644 --- a/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py +++ b/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py @@ -3,13 +3,13 @@ from __future__ import annotations from sqlbuild.adapter.contract.types import TypeDialect -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.contracts._helpers.native_contracts import native_model_contracts from sqlbuild.compiler.contracts.models import ContractValidationResult def evaluate_native_model_contracts( - *, project: CompiledProject, dialect: TypeDialect | str | None + *, project: CompiledProjectView, dialect: TypeDialect | str | None ) -> ContractValidationResult: """Return the contract diagnostics; Python evaluates only the models native defers.""" diff --git a/src/sqlbuild/compiler/contracts/main/_native_promotion_conflicts.py b/src/sqlbuild/compiler/contracts/main/_native_promotion_conflicts.py index 0327b490a2..0e7121f8c9 100644 --- a/src/sqlbuild/compiler/contracts/main/_native_promotion_conflicts.py +++ b/src/sqlbuild/compiler/contracts/main/_native_promotion_conflicts.py @@ -3,12 +3,13 @@ from __future__ import annotations from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.contracts._helpers.native_promotion import native_promotion_conflicts def native_promotion_conflict_diagnostics( - *, project: CompiledProject, adapter_default: TablePromotionMode, settings_file: str + *, project: CompiledProjectView, adapter_default: TablePromotionMode, settings_file: str ) -> tuple[CompilerDiagnostic, ...]: """Return the K011 promotion conflicts.""" diff --git a/src/sqlbuild/compiler/contracts/main/promotion_conflicts.py b/src/sqlbuild/compiler/contracts/main/promotion_conflicts.py index 5e0745a85d..dafebbdcf7 100644 --- a/src/sqlbuild/compiler/contracts/main/promotion_conflicts.py +++ b/src/sqlbuild/compiler/contracts/main/promotion_conflicts.py @@ -3,7 +3,8 @@ from __future__ import annotations from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.contracts.main._native_promotion_conflicts import ( native_promotion_conflict_diagnostics, ) @@ -11,7 +12,7 @@ def promotion_conflict_diagnostics( *, - project: CompiledProject, + project: CompiledProjectView, adapter_default: TablePromotionMode, settings_file: str, ) -> tuple[CompilerDiagnostic, ...]: diff --git a/src/sqlbuild/compiler/contracts/main/validate.py b/src/sqlbuild/compiler/contracts/main/validate.py index 548ac16eec..1f91939b3a 100644 --- a/src/sqlbuild/compiler/contracts/main/validate.py +++ b/src/sqlbuild/compiler/contracts/main/validate.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.adapter.contract.types import TypeDialect -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.contracts.main._evaluate_native_model_contracts import ( evaluate_native_model_contracts, ) @@ -12,7 +12,7 @@ def evaluate_model_contracts( *, - project: CompiledProject, + project: CompiledProjectView, dialect: TypeDialect | str | None = None, ) -> ContractValidationResult: """Evaluate model header column contracts against inferred output columns.""" diff --git a/src/sqlbuild/compiler/dag/_helpers/artifact.py b/src/sqlbuild/compiler/dag/_helpers/artifact.py index bd768286ac..886342867f 100644 --- a/src/sqlbuild/compiler/dag/_helpers/artifact.py +++ b/src/sqlbuild/compiler/dag/_helpers/artifact.py @@ -8,18 +8,21 @@ from sqlbuild.compiler.auditing.types import AuditSeverity from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledFunction, - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompiledSource, CompiledSqlScenario, CompiledSqlTest, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType +from sqlbuild.compiler.compile.types import ( + AttachedAuditTargetKind, + CompiledAuditView, + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompiledSeedView, + CompiledSourceView, +) from sqlbuild.compiler.dag.constants import REQUIRED_DAG_COLLECTION_KEYS from sqlbuild.compiler.dag.models import ( DagArtifact, @@ -85,7 +88,7 @@ def format_dag_json(*, artifact: DagArtifact) -> str: return json.dumps(_drop_none(asdict(artifact)), indent=2) -def _build_source_node(source: CompiledSource) -> DagNode: +def _build_source_node(source: CompiledSourceView) -> DagNode: entry: SourceEntry = source.source_entry target: DagTarget = DagTarget( database=entry.database, @@ -110,7 +113,7 @@ def _build_source_node(source: CompiledSource) -> DagNode: def _build_loader_node( *, - project: CompiledProject, + project: CompiledProjectView, loader: DiscoveredLoaderFunction, source_by_loader: dict[str, SourceEntry], ) -> DagNode: @@ -170,7 +173,7 @@ def _build_python_node(node: DiscoveredPythonNode) -> DagNode: ) -def _build_seed_node(seed: CompiledSeed) -> DagNode: +def _build_seed_node(seed: CompiledSeedView) -> DagNode: return DagNode( id=_node_id(seed.key), kind=NodeKind.SEED.value, @@ -185,7 +188,7 @@ def _build_seed_node(seed: CompiledSeed) -> DagNode: ) -def _build_function_node(function: CompiledFunction) -> DagNode: +def _build_function_node(function: CompiledFunctionView) -> DagNode: node_type: str = function_node_type(return_columns=function.return_columns) return DagNode( id=f"{node_type}:{function.name}", @@ -209,7 +212,7 @@ def _build_function_node(function: CompiledFunction) -> DagNode: ) -def _build_model_node(model: CompiledModel) -> DagNode: +def _build_model_node(model: CompiledModelView) -> DagNode: return DagNode( id=_node_id(model.key), kind=NodeKind.MODEL.value, @@ -229,18 +232,10 @@ def _build_model_node(model: CompiledModel) -> DagNode: def _build_edges( *, graph: ProjectGraph, python_graph: PythonNodeGraph | None = None ) -> tuple[DagEdge, ...]: - edges: list[DagEdge] = [] - key: CompiledObjectKey - dep_keys: tuple[CompiledObjectKey, ...] - for key, dep_keys in graph.upstream_deps.items(): - dep_key: CompiledObjectKey - for dep_key in dep_keys: - edges.append( - DagEdge( - from_id=_node_id(dep_key), - to_id=_node_id(key), - ) - ) + edges: list[DagEdge] = [ + DagEdge(from_id=f"{dep_kind}:{dep_name}", to_id=f"{kind}:{name}") + for (dep_kind, dep_name), (kind, name) in graph.native.edges() + ] edges.extend(_build_loader_edges(graph)) if python_graph is not None: edges.extend(_build_python_edges(python_graph)) @@ -368,7 +363,7 @@ def _sql_test_meta(test: CompiledSqlTest) -> dict[str, object]: } -def _build_audit_check(audit: CompiledAudit) -> DagCheck: +def _build_audit_check(audit: CompiledAuditView) -> DagCheck: checked_asset_ids: tuple[str, ...] = tuple(_node_id(key) for key in audit.scope_deps) if audit.attached_target_kind is not None and audit.attached_target_name is not None: resource_type: CompiledResourceType = AttachedAuditTargetKind( @@ -392,7 +387,7 @@ def _build_audit_check(audit: CompiledAudit) -> DagCheck: ) -def _audit_check_id(audit: CompiledAudit) -> str: +def _audit_check_id(audit: CompiledAuditView) -> str: parts: tuple[str | None, ...] = ( _node_id(audit.key), audit.attached_target_kind.value if audit.attached_target_kind is not None else None, @@ -507,7 +502,7 @@ def _source_by_loader(graph: ProjectGraph) -> dict[str, SourceEntry]: def _loader_to_source_entry( - *, project: CompiledProject, loader: DiscoveredLoaderFunction + *, project: CompiledProjectView, loader: DiscoveredLoaderFunction ) -> SourceEntry: destination: LoaderDestinationParts = loader_destination_parts( destination=( @@ -551,7 +546,7 @@ def _source_column(column: SourceColumnEntry) -> DagColumn: ) -def _model_columns(model: CompiledModel) -> tuple[DagColumn, ...]: +def _model_columns(model: CompiledModelView) -> tuple[DagColumn, ...]: if model.schema_entry is not None and model.schema_entry.columns: return tuple(_schema_column(column) for column in model.schema_entry.columns) if model.inferred_columns is not None: @@ -561,7 +556,7 @@ def _model_columns(model: CompiledModel) -> tuple[DagColumn, ...]: return () -def _model_tags(model: CompiledModel) -> tuple[str, ...]: +def _model_tags(model: CompiledModelView) -> tuple[str, ...]: schema_tags: tuple[str, ...] = () if model.schema_entry is not None: schema_tags = model.schema_entry.tags diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py index d06ee98589..2b80061b1d 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/declaration_files.py @@ -19,8 +19,6 @@ native_request, seed_snapshot_listings, ) -from sqlbuild.compiler.discovery._helpers.sql.declarations import typed_constant_declaration -from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values from sqlbuild.compiler.discovery._helpers.sql.schema_columns import parse_schema_columns from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( @@ -287,7 +285,7 @@ def _optional_path(text: str | None) -> Path | None: def _audit_block(block: tuple[object, ...]) -> DiscoveredAuditBlock: audit_index, values, sql_body, name, mode, measure_sql, evidence_sql = block - header_values: dict[str, object] = project_native_header_values(cast(dict[str, object], values)) + header_values: dict[str, object] = cast(dict[str, object], values) evaluation_mode: AuditEvaluationMode = AuditEvaluationMode(str(mode)) return DiscoveredAuditBlock( audit_index=cast(int, audit_index), @@ -331,7 +329,7 @@ def _enum_file( def _enum_declaration(*, relative_path: Path, declaration: _EnumPayload) -> EnumDeclaration: name, members, scalar_type = declaration - projected: dict[str, object] = project_native_header_values(members) + projected: dict[str, object] = members return EnumDeclaration( name=name, members=tuple( @@ -372,14 +370,11 @@ def _constant_declaration( *, file_path: Path, relative_path: Path, declaration: _ConstantPayload ) -> ConstantDeclaration: name, value, explicit_type, render_as = declaration - return typed_constant_declaration( - name=name, - value=project_native_header_values(value)["value"], - explicit_type=explicit_type, - render_as_text=render_as, - file_path=file_path, - relative_path=relative_path, - model_name=None, + return cast( + ConstantDeclaration, + _native.typed_constant_record( + (name, value["value"]), (explicit_type, render_as), (file_path, relative_path) + ), ) @@ -417,7 +412,7 @@ def _schema_declaration( description=description, extends=extends, columns=parse_schema_columns( - raw_columns=project_native_header_values(columns).get("columns"), + raw_columns=columns.get("columns"), file_path=file_path, label=f"schema '{name}'", error_class=DeclarationParseError, @@ -441,7 +436,7 @@ def _function_file( file_path=project_dir / relative_path, relative_path=relative_path, contents=str(contents), - header_values=project_native_header_values(cast(dict[str, object], header_values)), + header_values=cast(dict[str, object], header_values), body_sql=str(body_sql), ) @@ -458,7 +453,7 @@ def _hook_file( file_path=project_dir / relative_path, relative_path=relative_path, contents=str(contents), - header_values=project_native_header_values(cast(dict[str, object], header_values)), + header_values=cast(dict[str, object], header_values), sql_body=str(sql_body), name=str(name), description=cast(str | None, description), diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py index 4c166f2493..be9389a8f3 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py @@ -16,17 +16,17 @@ native_request, seed_snapshot_listings, ) -from sqlbuild.compiler.discovery._helpers.sql.declarations import ( - parse_model_constant_declarations, - parse_model_enum_declarations, -) -from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( REMOVED_SQL_MODEL_HEADER_KEYS, SQL_MODEL_HEADER_KEYS, ) -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveryFileFault +from sqlbuild.compiler.discovery.models import ( + ConstantDeclaration, + DiscoveredSqlModelFile, + DiscoveryFileFault, + EnumDeclaration, +) from sqlbuild.compiler.discovery.types import NativeFiles, NativeLocation @@ -87,7 +87,15 @@ def _model_file( if error is not None: raise error _tag, contents, values, header_locations, output_locations, query_sql = payload - header_values: dict[str, object] = project_native_header_values(cast(dict[str, object], values)) + header_values: dict[str, object] = cast(dict[str, object], values) + enum_declarations, constant_declarations = cast( + tuple[tuple[EnumDeclaration, ...], tuple[ConstantDeclaration, ...]], + _native.model_header_declarations( + (header_values.get("enums"), header_values.get("constants")), + file_path.stem, + relative_path, + ), + ) return DiscoveredSqlModelFile( file_path=file_path, relative_path=relative_path, @@ -100,15 +108,7 @@ def _model_file( locations=cast(list[NativeLocation], output_locations), relative_path=relative_path ), query_sql=str(query_sql), - enum_declarations=parse_model_enum_declarations( - raw_value=header_values.get("enums"), - model_name=file_path.stem, - relative_path=relative_path, - ), - constant_declarations=parse_model_constant_declarations( - raw_value=header_values.get("constants"), - model_name=file_path.stem, - relative_path=relative_path, - ), + enum_declarations=enum_declarations, + constant_declarations=constant_declarations, extract_implicit_alias_columns=extract_implicit_alias_columns, ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py index da39899a51..90cd6a3d9d 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/sql_test_files.py @@ -17,12 +17,7 @@ native_text_runtime, seed_snapshot_listings, ) -from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values from sqlbuild.compiler.discovery._helpers.sql.scenarios import build_sql_scenario_file -from sqlbuild.compiler.discovery._helpers.sql.tests import ( - build_sql_test_block, - validate_sql_test_names, -) from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot from sqlbuild.compiler.discovery.constants import ( SQL_SCENARIO_HEADER_KEYS, @@ -162,18 +157,13 @@ def _test_blocks( block_failure: tuple[object, ...] | None, ) -> tuple[DiscoveredSqlTestBlock, ...]: blocks: tuple[DiscoveredSqlTestBlock, ...] = tuple( - build_sql_test_block( - header_values=project_native_header_values(values), - sql_body=sql_body, - file_path=file_path, - test_index=test_index, - sql_body_span=sql_body_span, - ) - for test_index, (values, sql_body, sql_body_span) in enumerate(native_blocks, start=1) + cast(list[DiscoveredSqlTestBlock], _native.sql_test_blocks(str(file_path), native_blocks)) ) if block_failure is not None: raise native_failure(block_failure) - validate_sql_test_names(file_path=file_path, blocks=blocks) + _native.validate_sql_test_block_names( + str(file_path), [(block.test_index, block.name) for block in blocks] + ) return blocks @@ -193,7 +183,7 @@ def _scenario_file_from_payload( raise error _tag, contents, values, sql_body, sql_body_span = payload return build_sql_scenario_file( - header_values=project_native_header_values(cast(dict[str, object], values)), + header_values=cast(dict[str, object], values), sql_body=str(sql_body), sql_body_span=cast(tuple[int, int], sql_body_span), contents=str(contents), diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py b/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py deleted file mode 100644 index 6549ea85c7..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/declarations.py +++ /dev/null @@ -1,330 +0,0 @@ -"""Parsing helpers for authored public SQL declaration files.""" - -from __future__ import annotations - -import re -from collections.abc import Sequence -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.discovery.exceptions import DeclarationParseError -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - EnumDeclaration, - EnumMember, -) -from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( - validate_resource_identity, -) -from sqlbuild.sql_values.constants import MISSING_SQL_VALUE -from sqlbuild.sql_values.exceptions import SqlValueValidationError -from sqlbuild.sql_values.main.normalize import normalize_sql_value -from sqlbuild.sql_values.models import AuthoredSqlValueCall, SqlValue -from sqlbuild.sql_values.types import CollectionRendering, SqlValueKind - -_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -_VARCHAR_TYPE: str = "VARCHAR" -_INTEGER_TYPE: str = "INTEGER" -_STATEMENT_TERMINATOR: str = ";" -_ESCAPE_CHARACTER: str = "\\" - - -def parse_model_enum_declarations( - *, raw_value: object | None, model_name: str, relative_path: Path -) -> tuple[EnumDeclaration, ...]: - """Normalize model-header enum declarations into scoped facts.""" - - if raw_value is None: - return () - if not isinstance(raw_value, dict): - raise DeclarationParseError(f"{relative_path} model 'enums' must be a mapping") - declarations: list[EnumDeclaration] = [] - raw_name: object - raw_members: object - for raw_name, raw_members in cast(dict[object, object], raw_value).items(): - declarations.append( - _parse_enum_declaration( - values={"name": raw_name, "members": raw_members}, - file_path=relative_path, - relative_path=relative_path, - model_name=model_name, - ) - ) - return tuple(declarations) - - -def parse_model_constant_declarations( - *, raw_value: object | None, model_name: str, relative_path: Path -) -> tuple[ConstantDeclaration, ...]: - """Normalize model-header constant declarations into scoped facts.""" - - if raw_value is None: - return () - if not isinstance(raw_value, dict): - raise DeclarationParseError(f"{relative_path} model 'constants' must be a mapping") - declarations: list[ConstantDeclaration] = [] - raw_name: object - raw_value_item: object - for raw_name, raw_value_item in cast(dict[object, object], raw_value).items(): - declarations.append( - _parse_constant_declaration( - name=raw_name, - value=raw_value_item, - explicit_type=MISSING_SQL_VALUE, - raw_render_as=MISSING_SQL_VALUE, - file_path=relative_path, - relative_path=relative_path, - model_name=model_name, - unknown_keys=set(), - ) - ) - return tuple(declarations) - - -def _parse_enum_declaration( - *, - values: dict[str, object], - file_path: Path, - relative_path: Path, - model_name: str | None, -) -> EnumDeclaration: - unknown_keys: set[str] = set(values) - {"name", "members"} - if unknown_keys: - raise DeclarationParseError( - f"{file_path} enum has unknown keys: {', '.join(sorted(unknown_keys))}" - ) - name: str = _parse_declaration_name( - raw_name=values.get("name"), - file_path=file_path, - kind="enum", - model_name=model_name, - ) - raw_members: object | None = values.get("members") - members: tuple[EnumMember, ...] - if isinstance(raw_members, list): - members = _parse_shorthand_members( - raw_members=raw_members, - file_path=file_path, - enum_name=name, - ) - elif isinstance(raw_members, dict): - members = _parse_explicit_members( - raw_members=cast(dict[object, object], raw_members), - file_path=file_path, - enum_name=name, - ) - else: - raise DeclarationParseError(f"{file_path} enum '{name}' members must use [...] or (...)") - if not members: - raise DeclarationParseError(f"{file_path} enum '{name}' must declare at least one member") - invalid_member: EnumMember | None = next( - (member for member in members if member.name != member.name.upper()), - None, - ) - if invalid_member is not None: - raise DeclarationParseError( - f"{file_path} enum '{name}' member identifiers must be uppercase: " - f"'{invalid_member.name}'" - ) - member_types: set[type[object]] = {type(member.value) for member in members} - if len(member_types) != 1: - raise DeclarationParseError( - f"{file_path} enum '{name}' members must use one consistent scalar type" - ) - return EnumDeclaration( - name=name, - members=members, - scalar_type=_scalar_type(value=members[0].value), - relative_path=relative_path, - model_name=model_name, - ) - - -def _parse_shorthand_members( - *, raw_members: Sequence[object], file_path: Path, enum_name: str -) -> tuple[EnumMember, ...]: - members: list[EnumMember] = [] - seen_names: set[str] = set() - raw_member: object - for raw_member in raw_members: - if not isinstance(raw_member, str) or not _IDENTIFIER_PATTERN.fullmatch(raw_member): - raise DeclarationParseError( - f"{file_path} enum '{enum_name}' shorthand members must be identifiers" - ) - if raw_member in seen_names: - raise DeclarationParseError( - f"{file_path} enum '{enum_name}' has duplicate member '{raw_member}'" - ) - seen_names.add(raw_member) - members.append(EnumMember(name=raw_member, value=raw_member)) - return tuple(members) - - -def _parse_explicit_members( - *, raw_members: dict[object, object], file_path: Path, enum_name: str -) -> tuple[EnumMember, ...]: - members: list[EnumMember] = [] - raw_name: object - raw_value: object - for raw_name, raw_value in raw_members.items(): - member_name: str = _parse_identifier( - raw_value=raw_name, - file_path=file_path, - label=f"enum '{enum_name}' member", - ) - scalar_value: str | int = _parse_scalar( - raw_value=raw_value, - file_path=file_path, - label=f"enum '{enum_name}' member '{member_name}'", - ) - members.append(EnumMember(name=member_name, value=scalar_value)) - return tuple(members) - - -def _parse_constant_declaration( - *, - name: object | None, - value: object | None, - explicit_type: object | None, - raw_render_as: object | None, - file_path: Path, - relative_path: Path, - model_name: str | None, - unknown_keys: set[str], -) -> ConstantDeclaration: - if unknown_keys: - raise DeclarationParseError( - f"{file_path} constant has unknown keys: {', '.join(sorted(unknown_keys))}" - ) - parsed_name: str = _parse_declaration_name( - raw_name=name, - file_path=file_path, - kind="constant", - model_name=model_name, - ) - if isinstance(value, AuthoredSqlValueCall): - if explicit_type is not MISSING_SQL_VALUE or raw_render_as is not MISSING_SQL_VALUE: - raise DeclarationParseError( - f"{file_path} constant '{parsed_name}' cannot combine wrapper and outer options" - ) - wrapper_values: dict[str, object] = dict(value.arguments) - wrapper_unknown_keys: set[str] = set(wrapper_values) - {"value", "type", "render_as"} - if wrapper_unknown_keys: - raise DeclarationParseError( - f"{file_path} constant '{parsed_name}' wrapper has unknown keys: " - f"{', '.join(sorted(wrapper_unknown_keys))}" - ) - value = wrapper_values.get("value", MISSING_SQL_VALUE) - explicit_type = wrapper_values.get("type", MISSING_SQL_VALUE) - raw_render_as = wrapper_values.get("render_as", MISSING_SQL_VALUE) - if value is MISSING_SQL_VALUE: - raise DeclarationParseError( - f"{file_path} constant '{parsed_name}' is missing required value" - ) - parsed_explicit_type: str | None = _parse_constant_option( - raw_value=explicit_type, - file_path=file_path, - label=f"constant '{parsed_name}' type", - ) - render_as_text: str | None = _parse_constant_option( - raw_value=raw_render_as, - file_path=file_path, - label=f"constant '{parsed_name}' render_as", - ) - return typed_constant_declaration( - name=parsed_name, - value=value, - explicit_type=parsed_explicit_type, - render_as_text=render_as_text, - file_path=file_path, - relative_path=relative_path, - model_name=model_name, - ) - - -def typed_constant_declaration( - *, - name: str, - value: object, - explicit_type: str | None, - render_as_text: str | None, - file_path: Path, - relative_path: Path, - model_name: str | None, -) -> ConstantDeclaration: - """Normalize a constant whose name and options are valid into its typed declaration.""" - - parsed_name: str = name - try: - render_as: CollectionRendering | None = ( - CollectionRendering(render_as_text) if render_as_text is not None else None - ) - except ValueError as error: - raise DeclarationParseError( - f"{file_path} constant '{parsed_name}' render_as must be value_list or array" - ) from error - try: - typed_value: SqlValue = normalize_sql_value( - raw_value=value, - explicit_type=explicit_type, - context=f"{file_path} constant '{parsed_name}'", - ) - except SqlValueValidationError as error: - raise DeclarationParseError(str(error)) from error - if render_as is not None and typed_value.kind not in {SqlValueKind.LIST, SqlValueKind.SET}: - raise DeclarationParseError( - f"{file_path} constant '{parsed_name}' is {typed_value.kind.value} and does not " - "support " - f"render_as {render_as.value}" - ) - return ConstantDeclaration( - name=parsed_name, - value=typed_value, - relative_path=relative_path, - model_name=model_name, - render_as=render_as, - ) - - -def _parse_constant_option(*, raw_value: object, file_path: Path, label: str) -> str | None: - if raw_value is MISSING_SQL_VALUE: - return None - if not isinstance(raw_value, str) or not raw_value: - raise DeclarationParseError(f"{file_path} {label} must be an identifier") - return raw_value - - -def _parse_declaration_name( - *, raw_name: object | None, file_path: Path, kind: str, model_name: str | None -) -> str: - name: str = _parse_identifier(raw_value=raw_name, file_path=file_path, label=kind) - validate_resource_identity( - name=name, - kind=f"{'model-local' if model_name is not None else 'public'} {kind}", - path=file_path, - private_identity=model_name is not None, - ) - return name - - -def _parse_identifier(*, raw_value: object | None, file_path: Path, label: str) -> str: - if not isinstance(raw_value, str) or not _IDENTIFIER_PATTERN.fullmatch(raw_value): - raise DeclarationParseError(f"{file_path} {label} name must be a SQL identifier") - return raw_value - - -def _parse_scalar(*, raw_value: object | None, file_path: Path, label: str) -> str | int: - if isinstance(raw_value, bool) or not isinstance(raw_value, str | int): - raise DeclarationParseError(f"{file_path} {label} value must be a string or integer") - return raw_value - - -def _scalar_type(*, value: str | int) -> str: - return _VARCHAR_TYPE if isinstance(value, str) else _INTEGER_TYPE - - -def _skip_whitespace(*, contents: str, start: int) -> int: - index: int = start - while index < len(contents) and contents[index].isspace(): - index += 1 - return index diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/header_keys.py b/src/sqlbuild/compiler/discovery/_helpers/sql/header_keys.py index e99ca9bdf4..7b31d20f99 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/header_keys.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/header_keys.py @@ -2,43 +2,12 @@ from __future__ import annotations -import re from collections.abc import Iterable -from pathlib import Path -from sqlbuild.compiler.discovery._helpers.validation.supported_keys import unsupported_keys_help +import sqlbuild._native as _native from sqlbuild.compiler.discovery.exceptions import DiscoveryError -def reject_unsupported_header_keys( - *, - header_values: dict[str, object], - supported_keys: frozenset[str], - statement: str, - header: str, - header_line: int, - file_path: Path, - error_class: type[DiscoveryError], -) -> None: - """Raise when a header declares keys outside its set; `header_line` is 1-based.""" - - unsupported: tuple[str, ...] = tuple( - str(key) for key in header_values if key not in supported_keys - ) - if not unsupported: - return - line: int = header_line + header.count( - "\n", 0, _header_key_offset(header=header, key=unsupported[0]) - ) - raise unsupported_keys_error( - statement=statement, - location=f"{file_path}:{line}", - keys=unsupported, - supported_keys=supported_keys, - error_class=error_class, - ) - - def unsupported_keys_error( *, statement: str, @@ -52,12 +21,5 @@ def unsupported_keys_error( unsupported: tuple[str, ...] = tuple(keys) return error_class( f"{statement} in '{location}' has unsupported keys: {', '.join(unsupported)}", - help=unsupported_keys_help(keys=unsupported, supported_keys=supported_keys), - ) - - -def _header_key_offset(*, header: str, key: str) -> int: - match: re.Match[str] | None = re.search( - rf"(?:^|[(,])(?:\s|--[^\n]*(?:\n|$)|/\*.*?\*/)*({re.escape(key)})\b", header, re.S + help=_native.unsupported_keys_help(list(unsupported), list(supported_keys)), ) - return match.start(1) if match is not None else 0 diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py index 9da71444c9..2348239936 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py @@ -1,253 +1,49 @@ -"""Parsing helpers for authored SQL model files.""" +"""Native MODEL header parsing and authored-location queries for SQL model files.""" from __future__ import annotations -import re -from bisect import bisect_right from collections import OrderedDict -from collections.abc import Callable -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path -from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.discovery.constants import ( - STATEMENT_HEADER_BODY_PATTERN, -) from sqlbuild.compiler.discovery.exceptions import ( DiscoveryError, ModelHeaderSyntaxError, ModelSqlParseError, ) -from sqlbuild.compiler.discovery.models import ( - ModelHeaderColumnSpan, - ModelHeaderMatch, - NamedSqlHookEntry, - PythonHookEntry, - SqlHookEntry, -) from sqlbuild.spec.contracts.models import SourceLocation -from sqlbuild.sql_values.models import AuthoredSqlSet, AuthoredSqlValueCall -_MODEL_HEADER_END_TOKEN: str = "end" -_MODEL_HEADER_WORD_TOKEN: str = "word" -_MODEL_HEADER_STRING_TOKEN: str = "string" -_MODEL_HEADER_SYMBOL_TOKEN: str = "symbol" -_MODEL_HEADER_OPEN_PAREN: str = "(" -_MODEL_HEADER_CLOSE_PAREN: str = ")" -_MODEL_HEADER_COMMA: str = "," -_MODEL_HEADER_QUOTE_NAMES: dict[str, str] = {"'": "single", '"': "double"} -_MODEL_HEADER_ESCAPE_CHARACTER: str = "\\" -_MODEL_HEADER_COLUMNS_KEY: str = "columns" -_MODEL_HEADER_TRUE_VALUE: str = "true" -_MODEL_HEADER_FALSE_VALUE: str = "false" -_MODEL_HEADER_NULL_VALUE: str = "null" -_SQL_IDENTIFIER_SEPARATOR: str = "_" -_SQL_SELECT_KEYWORD: str = "SELECT" -_SQL_UNION_KEYWORD: str = "UNION" -_SQL_FROM_KEYWORD: str = "FROM" -_SQL_SELECT_INITIAL: str = _SQL_SELECT_KEYWORD[0] -_SQL_UNION_INITIAL: str = _SQL_UNION_KEYWORD[0] -_SQL_FROM_INITIAL: str = _SQL_FROM_KEYWORD[0] -_SQL_UNION_LOWER_KEYWORD: str = _SQL_UNION_KEYWORD.lower() -_MODEL_HEADER_INTEGER_PATTERN: re.Pattern[str] = re.compile(r"^[+-]?\d+$") -_MODEL_HEADER_FLOAT_PATTERN: re.Pattern[str] = re.compile(r"^[+-]?(?:\d+\.\d*|\d*\.\d+)$") _MODEL_HEADER_TOKEN_CACHE_SIZE: int = 4096 -_PROJECTION_PENDING: object = object() -_MODEL_HEADER_NESTING_ERROR_PATTERN: re.Pattern[str] = re.compile( - r"values nest deeper than (?P\d+) levels at position (?P\d+)" -) -_NATIVE_MODEL_HEADER_END_TOKEN: int = 0 -_NATIVE_MODEL_HEADER_WORD_TOKEN: int = 1 -_NATIVE_MODEL_HEADER_STRING_TOKEN: int = 2 -_NATIVE_MODEL_HEADER_SYMBOL_TOKEN: int = 3 -_NATIVE_MARKER_LENGTH: int = 2 -_NATIVE_SET_MARKER: str = "set" -_NATIVE_TUPLE_MARKER: str = "tuple" -_NATIVE_CONSTANT_MARKER: str = "constant" -_NATIVE_INLINE_SQL_MARKER: str = "inline_sql" -_NATIVE_SQL_MARKER: str = "sql" -_NATIVE_PYTHON_MARKER: str = "python" -_MODEL_HEADER_TOKEN_KIND_BY_NATIVE: dict[int, str] = { - _NATIVE_MODEL_HEADER_END_TOKEN: _MODEL_HEADER_END_TOKEN, - _NATIVE_MODEL_HEADER_WORD_TOKEN: _MODEL_HEADER_WORD_TOKEN, - _NATIVE_MODEL_HEADER_STRING_TOKEN: _MODEL_HEADER_STRING_TOKEN, - _NATIVE_MODEL_HEADER_SYMBOL_TOKEN: _MODEL_HEADER_SYMBOL_TOKEN, -} -_MODEL_HEADER_PATTERN: re.Pattern[str] = re.compile( - r"^\s*MODEL\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;\s*(?P.*)\Z", - re.DOTALL, -) -_SELECT_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"['\"`()sSfFuU]") - - -@dataclass -class _ProjectionFrame: - """One container being projected: its native children, projected so far, and its builder.""" - - children: tuple[object, ...] - build: Callable[[list[object]], object] - projected: list[object] = field(default_factory=list) @dataclass(frozen=True) -class _ModelHeaderToken: - kind: str - value: str - position: int - - -@dataclass(frozen=True) -class _ModelHeaderTokenization: +class _ModelHeaderParse: values: dict[str, object] | None - column_relative_locations: tuple[tuple[str, int, int, int], ...] | None error: str | None -class _ModelHeaderTokenCache: +class _ModelHeaderParseCache: def __init__(self) -> None: - self._values: OrderedDict[str, _ModelHeaderTokenization] = OrderedDict() + self._values: OrderedDict[str, _ModelHeaderParse] = OrderedDict() def missing(self, headers: list[str]) -> list[str]: return list(dict.fromkeys(header for header in headers if header not in self._values)) - def get(self, header: str) -> _ModelHeaderTokenization | None: - tokenization: _ModelHeaderTokenization | None = self._values.get(header) - if tokenization is not None: + def get(self, header: str) -> _ModelHeaderParse | None: + parsed: _ModelHeaderParse | None = self._values.get(header) + if parsed is not None: self._values.move_to_end(header) - return tokenization + return parsed - def put(self, *, header: str, tokenization: _ModelHeaderTokenization) -> None: - self._values[header] = tokenization + def put(self, *, header: str, parsed: _ModelHeaderParse) -> None: + self._values[header] = parsed self._values.move_to_end(header) if len(self._values) > _MODEL_HEADER_TOKEN_CACHE_SIZE: self._values.popitem(last=False) -_MODEL_HEADER_TOKEN_CACHE: _ModelHeaderTokenCache = _ModelHeaderTokenCache() - - -def match_model_header(contents: str) -> ModelHeaderMatch | None: - """Match the MODEL(...) header once so callers can reuse it for every header projection.""" - - header_match: re.Match[str] | None = _MODEL_HEADER_PATTERN.match(contents) - if header_match is None: - return None - return ModelHeaderMatch( - contents=contents, - header_start=header_match.start("header"), - header_end=header_match.end("header"), - sql_start=header_match.start("sql"), - ) - - -def model_header_columns_span( - *, contents: str -) -> tuple[int, int, dict[str, ModelHeaderColumnSpan]] | None: - """Return the columns body and column entry spans from the native header tokenizer.""" - - header_match: re.Match[str] | None = _MODEL_HEADER_PATTERN.match(contents) - if header_match is None: - return None - header: str = header_match.group("header") - header_start: int = header_match.start("header") - tokens: list[_ModelHeaderToken] = _tokenize_model_header_for_spans(header) - columns_open_index: int | None = None - depth: int = 0 - for index, token in enumerate(tokens): - if ( - token.kind == _MODEL_HEADER_WORD_TOKEN - and token.value == _MODEL_HEADER_COLUMNS_KEY - and depth == 0 - and tokens[index + 1].value == _MODEL_HEADER_OPEN_PAREN - ): - columns_open_index = index + 1 - break - if token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == _MODEL_HEADER_OPEN_PAREN: - depth += 1 - elif token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == _MODEL_HEADER_CLOSE_PAREN: - depth -= 1 - if columns_open_index is None: - return None - open_token: _ModelHeaderToken = tokens[columns_open_index] - spans: dict[str, ModelHeaderColumnSpan] = {} - depth = 1 - index: int = columns_open_index + 1 - close_token: _ModelHeaderToken | None = None - while index < len(tokens): - token: _ModelHeaderToken = tokens[index] - if token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == _MODEL_HEADER_OPEN_PAREN: - depth += 1 - elif token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == _MODEL_HEADER_CLOSE_PAREN: - depth -= 1 - if depth == 0: - close_token = token - break - elif token.kind == _MODEL_HEADER_WORD_TOKEN and depth == 1: - metadata_open: _ModelHeaderToken = tokens[index + 1] - if metadata_open.value == _MODEL_HEADER_OPEN_PAREN: - metadata_depth: int = 1 - end_index: int = index + 2 - while end_index < len(tokens): - candidate: _ModelHeaderToken = tokens[end_index] - if candidate.value == _MODEL_HEADER_OPEN_PAREN: - metadata_depth += 1 - elif candidate.value == _MODEL_HEADER_CLOSE_PAREN: - metadata_depth -= 1 - if metadata_depth == 0: - spans[token.value] = ModelHeaderColumnSpan( - entry_start=header_start + token.position, - entry_end=header_start + candidate.position + 1, - metadata_start=header_start + metadata_open.position + 1, - metadata_end=header_start + candidate.position, - ) - index = end_index - break - end_index += 1 - index += 1 - if close_token is None: - return None - return ( - header_start + open_token.position + 1, - header_start + close_token.position, - spans, - ) - - -def model_header_entry_span(*, contents: str, key: str) -> tuple[int, int] | None: - """Return authored byte offsets for one top-level `key value` MODEL header entry.""" - - header_match: re.Match[str] | None = _MODEL_HEADER_PATTERN.match(contents) - if header_match is None: - return None - header_start: int = header_match.start("header") - tokens: list[_ModelHeaderToken] = _tokenize_model_header_for_spans(header_match.group("header")) - depth: int = 0 - for index, token in enumerate(tokens): - if token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == _MODEL_HEADER_OPEN_PAREN: - depth += 1 - elif token.kind == _MODEL_HEADER_SYMBOL_TOKEN and token.value == _MODEL_HEADER_CLOSE_PAREN: - depth -= 1 - elif ( - depth == 0 - and token.kind == _MODEL_HEADER_WORD_TOKEN - and token.value == key - and index + 2 < len(tokens) - ): - entry_end: int = header_start + tokens[index + 2].position - while entry_end > header_start + token.position and contents[entry_end - 1].isspace(): - entry_end -= 1 - return header_start + token.position, entry_end - return None - - -def model_header_body_span(*, contents: str) -> tuple[int, int] | None: - """Return authored byte offsets for the MODEL header body.""" - - header_match: re.Match[str] | None = _MODEL_HEADER_PATTERN.match(contents) - if header_match is None: - return None - return header_match.start("header"), header_match.end("header") +_MODEL_HEADER_PARSE_CACHE: _ModelHeaderParseCache = _ModelHeaderParseCache() def header_column_locations( @@ -255,26 +51,10 @@ def header_column_locations( ) -> dict[str, SourceLocation]: """Return authored column locations from a parsed SQLBuild header.""" - tokenization: _ModelHeaderTokenization = _model_header_parse(header) - if tokenization.column_relative_locations is None: - raise ModelHeaderSyntaxError("Native MODEL header parser returned no column locations") - header_line: int = contents.count("\n", 0, header_start) + 1 - preceding_newline: int = contents.rfind("\n", 0, header_start) - header_column: int = ( - header_start + 1 if preceding_newline < 0 else header_start - preceding_newline - ) - return { - name: SourceLocation( - path=relative_path, - line=header_line + relative_line - 1, - column=relative_column + (header_column - 1 if relative_line == 1 else 0), - end_line=header_line + relative_line - 1, - end_column=( - relative_column + (header_column - 1 if relative_line == 1 else 0) + length - ), - ) - for name, relative_line, relative_column, length in tokenization.column_relative_locations - } + locations, error = _native.model_header_column_locations(contents, header, header_start) + if locations is None: + raise ModelHeaderSyntaxError(str(error)) + return _source_locations(rows=locations, relative_path=relative_path) def model_output_column_locations( @@ -282,287 +62,21 @@ def model_output_column_locations( ) -> dict[str, SourceLocation]: """Return authored locations for simple SELECT output expressions.""" - return matched_model_output_column_locations( - contents=contents, - header_match=match_model_header(contents), + return _source_locations( + rows=_native.model_output_columns(contents, extract_implicit_alias_columns), relative_path=relative_path, - extract_implicit_alias_columns=extract_implicit_alias_columns, ) -def matched_model_output_column_locations( - *, - contents: str, - header_match: ModelHeaderMatch | None, - relative_path: Path, - extract_implicit_alias_columns: bool, +def _source_locations( + *, rows: list[tuple[str, int, int, int, int]], relative_path: Path ) -> dict[str, SourceLocation]: - """Return simple SELECT output locations from one precomputed header match.""" - - if header_match is None: - return {} - sql_start: int = header_match.sql_start - sql: str = header_match.sql - projection_ranges: tuple[tuple[int, int], ...] | None = _top_level_select_projection_ranges(sql) - if projection_ranges is None: - return {} - return _scanner_output_column_locations( - contents=contents, - sql_start=sql_start, - relative_path=relative_path, - projection_ranges=projection_ranges, - extract_implicit_alias_columns=extract_implicit_alias_columns, - line_starts=source_line_starts(contents), - ) - - -def _top_level_select_projection_ranges(sql: str) -> tuple[tuple[int, int], ...] | None: - bounds: tuple[int, int] | None = _top_level_select_list_bounds(sql) - if bounds is None: - return None - select_list_start, select_list_end = bounds - return _split_top_level_select_items(sql=sql, start=select_list_start, end=select_list_end) - - -def _top_level_select_list_bounds(sql: str) -> tuple[int, int] | None: - depth: int = 0 - index: int = 0 - select_list_start: int | None = None - select_list_end: int | None = None - in_quote: str | None = None - length: int = len(sql) - has_union_candidate: bool = _SQL_UNION_LOWER_KEYWORD in sql.lower() - while index < length: - if in_quote is None: - special: re.Match[str] | None = _SELECT_SCAN_SPECIAL.search(sql, index) - if special is None: - break - index = special.start() - character: str = sql[index] - if in_quote is not None: - if character == _MODEL_HEADER_ESCAPE_CHARACTER: - index += 2 - continue - if character == in_quote: - in_quote = None - index += 1 - continue - if character in _MODEL_HEADER_QUOTE_NAMES: - in_quote = character - index += 1 - continue - if character == _MODEL_HEADER_OPEN_PAREN: - depth += 1 - index += 1 - continue - if character == _MODEL_HEADER_CLOSE_PAREN: - depth = max(0, depth - 1) - index += 1 - continue - if depth != 0: - index += 1 - continue - upper_character: str = character.upper() - if select_list_start is None: - if upper_character == _SQL_SELECT_INITIAL and _keyword_at( - sql=sql, keyword=_SQL_SELECT_KEYWORD, index=index - ): - select_list_start = index + len(_SQL_SELECT_KEYWORD) - index = select_list_start - continue - else: - if upper_character == _SQL_UNION_INITIAL and _keyword_at( - sql=sql, keyword=_SQL_UNION_KEYWORD, index=index - ): - return None - if ( - select_list_end is None - and upper_character == _SQL_FROM_INITIAL - and _keyword_at(sql=sql, keyword=_SQL_FROM_KEYWORD, index=index) - ): - select_list_end = index - if not has_union_candidate: - return select_list_start, select_list_end - index += 1 - if select_list_start is None: - return None - return select_list_start, select_list_end if select_list_end is not None else length - - -def _scanner_output_column_locations( - *, - contents: str, - sql_start: int, - relative_path: Path, - projection_ranges: tuple[tuple[int, int], ...], - extract_implicit_alias_columns: bool, - line_starts: tuple[int, ...], -) -> dict[str, SourceLocation]: - locations: dict[str, SourceLocation] = {} - item_start: int - item_end: int - for item_start, item_end in projection_ranges: - output_name: str | None = _select_item_output_name( - item=contents[sql_start + item_start : sql_start + item_end], - extract_implicit_alias_columns=extract_implicit_alias_columns, - ) - if output_name is None: - continue - location: SourceLocation | None = _location_for_projection_range( - contents=contents, - sql_start=sql_start, - relative_path=relative_path, - projection_range=(item_start, item_end), - line_starts=line_starts, - ) - if location is not None: - locations[output_name] = location - return locations - - -def _location_for_projection_range( - *, - contents: str, - sql_start: int, - relative_path: Path, - projection_range: tuple[int, int], - line_starts: tuple[int, ...], -) -> SourceLocation | None: - item_start: int = projection_range[0] - item_end: int = projection_range[1] - sql: str = contents[sql_start:] - start_offset: int = _skip_local_whitespace(sql=sql, start=item_start, end=item_end) - end_offset: int = _trim_local_whitespace(sql=sql, start=start_offset, end=item_end) - if start_offset >= end_offset: - return None - return _location_for_absolute_span( - contents=contents, - start=sql_start + start_offset, - end=sql_start + end_offset, - relative_path=relative_path, - line_starts=line_starts, - ) - - -def _location_for_absolute_span( - *, contents: str, start: int, end: int, relative_path: Path, line_starts: tuple[int, ...] -) -> SourceLocation: - end = max(start, end) - end_position: int = max(start, end - 1) - line_index: int = bisect_right(line_starts, start) - 1 - line: int = line_index + 1 - line_start: int = line_starts[line_index] - column: int = start - line_start + 1 - end_line_index: int = bisect_right(line_starts, end_position) - 1 - end_line: int = end_line_index + 1 - end_line_start: int = line_starts[end_line_index] - end_column: int = end_position - end_line_start + 2 - return SourceLocation( - path=relative_path, - line=line, - column=column, - end_line=end_line, - end_column=end_column, - ) - - -def source_line_starts(contents: str) -> tuple[int, ...]: - """Return the offset where each line of contents starts.""" - - starts: list[int] = [0] - index: int = contents.find("\n") - while index != -1: - starts.append(index + 1) - index = contents.find("\n", index + 1) - return tuple(starts) - - -def source_position(*, offset: int, line_starts: tuple[int, ...]) -> tuple[int, int]: - """Map a zero-based offset to a one-based line and column.""" - - line_index: int = bisect_right(line_starts, offset) - 1 - return line_index + 1, offset - line_starts[line_index] + 1 - - -def _split_top_level_select_items(*, sql: str, start: int, end: int) -> tuple[tuple[int, int], ...]: - items: list[tuple[int, int]] = [] - depth: int = 0 - item_start: int = start - index: int = start - in_quote: str | None = None - while index < end: - character: str = sql[index] - if in_quote is not None: - if character == _MODEL_HEADER_ESCAPE_CHARACTER: - index += 2 - continue - if character == in_quote: - in_quote = None - index += 1 - continue - if character in _MODEL_HEADER_QUOTE_NAMES: - in_quote = character - index += 1 - continue - if character == _MODEL_HEADER_OPEN_PAREN: - depth += 1 - elif character == _MODEL_HEADER_CLOSE_PAREN: - depth = max(0, depth - 1) - elif character == _MODEL_HEADER_COMMA and depth == 0: - items.append((item_start, index)) - item_start = index + 1 - index += 1 - items.append((item_start, end)) - return tuple(items) - - -def _select_item_output_name(*, item: str, extract_implicit_alias_columns: bool) -> str | None: - as_match: re.Match[str] | None = re.search( - r"\s+AS\s+(?P[A-Za-z_][A-Za-z0-9_]*|\"[^\"]+\")\s*\Z", - item, - flags=re.IGNORECASE, - ) - if as_match is not None: - return as_match.group("name").strip('"') - bare_match: re.Match[str] | None = re.match( - r"\s*(?:(?:[A-Za-z_][A-Za-z0-9_]*|\"[^\"]+\")\.)?" - r"(?P[A-Za-z_][A-Za-z0-9_]*|\"[^\"]+\")\s*\Z", - item, - ) - if bare_match is None: - if not extract_implicit_alias_columns: - return None - implicit_alias_match: re.Match[str] | None = re.search( - r"\)\s+(?P[A-Za-z_][A-Za-z0-9_]*|\"[^\"]+\")\s*\Z", - item, + return { + name: SourceLocation( + path=relative_path, line=line, column=column, end_line=end_line, end_column=end_column ) - if implicit_alias_match is None: - return None - return implicit_alias_match.group("name").strip('"') - return bare_match.group("name").strip('"') - - -def _skip_local_whitespace(*, sql: str, start: int, end: int) -> int: - while start < end and sql[start].isspace(): - start += 1 - return start - - -def _trim_local_whitespace(*, sql: str, start: int, end: int) -> int: - while end > start and sql[end - 1].isspace(): - end -= 1 - return end - - -def _keyword_at(*, sql: str, keyword: str, index: int) -> bool: - end: int = index + len(keyword) - if sql[index:end].upper() != keyword: - return False - before: str = sql[index - 1] if index > 0 else " " - after: str = sql[end] if end < len(sql) else " " - return not (before.isalnum() or before == _SQL_IDENTIFIER_SEPARATOR) and not ( - after.isalnum() or after == _SQL_IDENTIFIER_SEPARATOR - ) + for name, line, column, end_line, end_column in rows + } def parse_header_values( @@ -575,233 +89,38 @@ def parse_header_values( ) -> dict[str, object]: """Parse one header; a nesting error names its file line when `header_line` is known.""" - try: - parsed: _ModelHeaderTokenization = _model_header_parse(header) - if parsed.values is None: - raise ModelHeaderSyntaxError( - "Native MODEL header parser returned neither values nor an error" - ) + parsed: _ModelHeaderParse = _model_header_parse(header) + if parsed.values is not None: return dict(parsed.values) - except ModelSqlParseError: - raise - except ModelHeaderSyntaxError as error: - nesting: re.Match[str] | None = _MODEL_HEADER_NESTING_ERROR_PATTERN.fullmatch(str(error)) - nesting_help: str | None = ( - None - if nesting is None - else f"flatten the value so it nests at most {nesting.group('limit')} levels deep" - ) - if nesting is None or header_line is None: - raise error_class( - f"{statement_name}(...) in '{file_path}' contains invalid SQLBuild header syntax: " - f"{error}", - help=nesting_help, - ) from error - line: int = header_line + header.count("\n", 0, int(nesting.group("position"))) - raise error_class( - f"{statement_name}(...) in '{file_path}:{line}' contains invalid SQLBuild header " - f"syntax: values nest deeper than {nesting.group('limit')} levels", - help=nesting_help, - ) from error + error: ModelHeaderSyntaxError = ModelHeaderSyntaxError(str(parsed.error)) + detail, help_text, line = _native.model_header_failure(str(parsed.error), header, header_line) + location: str = str(file_path) if line is None else f"{file_path}:{line}" + raise error_class( + f"{statement_name}(...) in '{location}' contains invalid SQLBuild header syntax: {detail}", + help=help_text, + ) from error def prepare_model_header_tokens(headers: list[str]) -> None: """Batch cache-missing MODEL header semantic parsing through the native parser.""" - missing_headers: list[str] = _MODEL_HEADER_TOKEN_CACHE.missing(headers) + missing_headers: list[str] = _MODEL_HEADER_PARSE_CACHE.missing(headers) if not missing_headers: return - tokenizations: list[ - tuple[ - dict[str, object] | None, - list[tuple[str, int, int]] | None, - str | None, - ] + parsed_headers: list[ + tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None] ] = _native.parse_model_headers(missing_headers) - if len(tokenizations) != len(missing_headers): - raise ModelHeaderSyntaxError("Native MODEL header tokenizer returned an incomplete batch") - for header, (native_values, column_offsets, error) in zip( - missing_headers, tokenizations, strict=True - ): - _cache_model_header_tokenization( - header=header, - tokenization=_ModelHeaderTokenization( - values=( - _project_native_header_map(native_values) if native_values is not None else None - ), - column_relative_locations=( - _header_column_relative_locations(header=header, offsets=column_offsets) - if column_offsets is not None - else None - ), - error=error, - ), - ) - - -def _header_column_relative_locations( - *, header: str, offsets: list[tuple[str, int, int]] -) -> tuple[tuple[str, int, int, int], ...]: - """Project ordered header offsets to one-based relative source locations in one pass.""" - - locations: list[tuple[str, int, int, int]] = [] - line: int = 1 - column: int = 1 - cursor: int = 0 - for name, position, length in offsets: - while cursor < position: - newline: int = header.find("\n", cursor, position) - if newline < 0: - column += position - cursor - cursor = position - else: - line += 1 - column = 1 - cursor = newline + 1 - locations.append((name, line, column, length)) - return tuple(locations) - - -def project_native_header_values(values: dict[str, object]) -> dict[str, object]: - """Project native header parser values, with their marker tuples, to Python header values.""" - - return _project_native_header_map(values) - - -def _project_native_header_map(values: dict[str, object]) -> dict[str, object]: - return cast(dict[str, object], _project_native_header_value(values)) - - -def _project_native_header_value(value: object) -> object: - """Project one native value depth-first with an explicit stack, never Python recursion.""" - - frames: list[_ProjectionFrame] = [] - node: object = value - while True: - expansion: _ProjectionFrame | None = _projection_frame(node) - if expansion is None: - projected: object = _project_native_leaf(node) - else: - frames.append(expansion) - projected = _PROJECTION_PENDING - while True: - if projected is not _PROJECTION_PENDING: - if not frames: - return projected - frames[-1].projected.append(projected) - frame: _ProjectionFrame = frames[-1] - if len(frame.projected) < len(frame.children): - node = frame.children[len(frame.projected)] - break - frames.pop() - projected = frame.build(frame.projected) - - -def _projection_frame(value: object) -> _ProjectionFrame | None: - if isinstance(value, dict): - return _map_frame(values=cast(dict[str, object], value), build=dict) - if isinstance(value, list): - return _ProjectionFrame(children=tuple(value), build=list) - if ( - not isinstance(value, tuple) - or len(value) != _NATIVE_MARKER_LENGTH - or not isinstance(value[0], str) - ): - return None - kind: str = value[0] - payload: object = value[1] - if kind == _NATIVE_SET_MARKER and isinstance(payload, list): - return _ProjectionFrame( - children=tuple(payload), build=lambda items: AuthoredSqlSet(tuple(items)) - ) - if kind == _NATIVE_TUPLE_MARKER and isinstance(payload, list): - return _ProjectionFrame(children=tuple(payload), build=tuple) - if kind == _NATIVE_CONSTANT_MARKER and isinstance(payload, dict): - return _map_frame( - values=cast(dict[str, object], payload), - build=lambda items: AuthoredSqlValueCall(arguments=tuple(items)), + for header, (values, _offsets, error) in zip(missing_headers, parsed_headers, strict=True): + _MODEL_HEADER_PARSE_CACHE.put( + header=header, parsed=_ModelHeaderParse(values=values, error=error) ) - if ( - kind in {_NATIVE_SQL_MARKER, _NATIVE_PYTHON_MARKER} - and isinstance(payload, tuple) - and len(payload) == _NATIVE_MARKER_LENGTH - ): - name, kwargs = payload - if isinstance(name, str) and isinstance(kwargs, dict): - entry_type: type[NamedSqlHookEntry] | type[PythonHookEntry] = ( - NamedSqlHookEntry if kind == _NATIVE_SQL_MARKER else PythonHookEntry - ) - return _map_frame( - values=cast(dict[str, object], kwargs), - build=lambda items: entry_type(name=name, kwargs=dict(items)), - ) - return None -def _map_frame( - *, values: dict[str, object], build: Callable[[list[tuple[str, object]]], object] -) -> _ProjectionFrame: - keys: tuple[str, ...] = tuple(values) - return _ProjectionFrame( - children=tuple(values.values()), - build=lambda projected: build(list(zip(keys, projected, strict=True))), - ) - - -def _project_native_leaf(value: object) -> object: - if ( - not isinstance(value, tuple) - or len(value) != _NATIVE_MARKER_LENGTH - or not isinstance(value[0], str) - ): - return value - kind: str = value[0] - payload: object = value[1] - if kind == _MODEL_HEADER_WORD_TOKEN and isinstance(payload, str): - return _parse_word_value(payload) - if kind == _NATIVE_INLINE_SQL_MARKER and isinstance(payload, str): - return SqlHookEntry(statement=payload) - raise ModelHeaderSyntaxError(f"Native MODEL header parser returned invalid '{kind}' marker") - - -def _model_header_parse(header: str) -> _ModelHeaderTokenization: - tokenization: _ModelHeaderTokenization | None = _MODEL_HEADER_TOKEN_CACHE.get(header) - if tokenization is None: +def _model_header_parse(header: str) -> _ModelHeaderParse: + parsed: _ModelHeaderParse | None = _MODEL_HEADER_PARSE_CACHE.get(header) + if parsed is None: prepare_model_header_tokens([header]) - tokenization = _MODEL_HEADER_TOKEN_CACHE.get(header) - if tokenization is None: - raise ModelHeaderSyntaxError("Native MODEL header parser did not populate its cache") - if tokenization.error is not None: - raise ModelHeaderSyntaxError(tokenization.error) - return tokenization - - -def _tokenize_model_header_for_spans(header: str) -> list[_ModelHeaderToken]: - return [ - _ModelHeaderToken( - kind=_MODEL_HEADER_TOKEN_KIND_BY_NATIVE[kind], - value=value, - position=position, - ) - for kind, value, position in _native.tokenize_model_header(header) - ] - - -def _cache_model_header_tokenization( - *, header: str, tokenization: _ModelHeaderTokenization -) -> None: - _MODEL_HEADER_TOKEN_CACHE.put(header=header, tokenization=tokenization) - - -def _parse_word_value(value: str) -> object: - if value == _MODEL_HEADER_TRUE_VALUE: - return True - if value == _MODEL_HEADER_FALSE_VALUE: - return False - if value == _MODEL_HEADER_NULL_VALUE: - return None - if _MODEL_HEADER_INTEGER_PATTERN.match(value): - return int(value) - if _MODEL_HEADER_FLOAT_PATTERN.match(value): - return float(value) - return value + parsed = _MODEL_HEADER_PARSE_CACHE.get(header) + if parsed is None: + raise ModelHeaderSyntaxError("Native MODEL header parser did not populate its cache") + return parsed diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py deleted file mode 100644 index 742e92c7f9..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Parsing helpers for authored SQL-native test files.""" - -from __future__ import annotations - -import re -from datetime import date, datetime -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.compile.constants import ( - DEFAULT_SQL_TEST_MODE, - MACRO_ACTUAL_TEST_CTE_NAME, - TABLE_FN_ACTUAL_TEST_CTE_NAME, - UDF_ACTUAL_TEST_CTE_NAME, -) -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.discovery.constants import ( - SQL_ANALYSIS_CONFIG_KEY, -) -from sqlbuild.compiler.discovery.exceptions import SqlTestParseError -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlTestBlock, - DiscoveredSqlTestCase, - SqlTestParameterDeclaration, -) -from sqlbuild.sql_values.exceptions import SqlValueValidationError -from sqlbuild.sql_values.main.normalize import normalize_sql_value -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind - -_MODE_BY_ACTUAL_CTE_NAME: dict[str, SqlTestMode] = { - MACRO_ACTUAL_TEST_CTE_NAME: SqlTestMode.MACRO, - UDF_ACTUAL_TEST_CTE_NAME: SqlTestMode.UDF, - TABLE_FN_ACTUAL_TEST_CTE_NAME: SqlTestMode.TABLE_FN, -} -_ACTUAL_CTE_NAME_MARKER: str = "_actual__" -_TOP_LEVEL_ACTUAL_CTE_TOKEN_PATTERN: re.Pattern[str] = re.compile( - r"--[^\n]*" - r"|/\*[\s\S]*?(?:\*/|\Z)" - r"|'(?:''|[^'])*(?:'|\Z)" - r'|"(?:""|[^"])*(?:"|\Z)' - r"|`[^`]*(?:`|\Z)" - r"|(?P\()" - r"|(?P\))" - r"|(?" - + "|".join(re.escape(name) for name in _MODE_BY_ACTUAL_CTE_NAME) - + r")(?![\w$])(?=\s*(?:\([^()]*\)\s*)?(?i:AS)(?![\w$]))" -) -_OPEN_PAREN: str = "(" -_CLOSE_PAREN: str = ")" -_TEST_NAME_HEADER_KEY: str = "name" -_TEST_MODE_HEADER_KEY: str = "mode" -_TEST_PARAMETERS_HEADER_KEY: str = "parameters" -_TEST_CASES_HEADER_KEY: str = "cases" -_TEST_CURSOR_START_HEADER_KEY: str = "cursor_start" -_TEST_CURSOR_END_HEADER_KEY: str = "cursor_end" -_PARAMETER_TYPES: tuple[SqlValueKind, ...] = ( - SqlValueKind.STRING, - SqlValueKind.INTEGER, - SqlValueKind.BOOLEAN, - SqlValueKind.FLOAT, - SqlValueKind.DECIMAL, -) -_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") - - -def _infer_sql_test_mode(*, sql_body: str, file_path: Path) -> SqlTestMode: - """Infer a TEST block's mode from the direct-logic actual CTE it defines, if any.""" - - if _ACTUAL_CTE_NAME_MARKER not in sql_body: - return DEFAULT_SQL_TEST_MODE - depth: int = 0 - inferred_modes: dict[SqlTestMode, None] = {} - match: re.Match[str] - for match in _TOP_LEVEL_ACTUAL_CTE_TOKEN_PATTERN.finditer(sql_body): - if match.group("open") is not None: - depth += 1 - elif match.group("close") is not None: - depth = max(depth - 1, 0) - elif match.group("name") is not None and depth == 0: - inferred_modes[_MODE_BY_ACTUAL_CTE_NAME[match.group("name")]] = None - if len(inferred_modes) > 1: - names: str = ", ".join(mode.value for mode in inferred_modes) - raise SqlTestParseError( - f"SQL test '{file_path}' defines actual CTEs for several test modes ({names}); " - "each TEST block tests one kind of logic" - ) - return next(iter(inferred_modes), DEFAULT_SQL_TEST_MODE) - - -def build_sql_test_block( - *, - header_values: dict[str, object], - sql_body: str, - file_path: Path, - test_index: int, - sql_body_span: tuple[int, int] | None = None, -) -> DiscoveredSqlTestBlock: - """Validate a header-parsed TEST block with only supported keys and build its record.""" - - if _TEST_NAME_HEADER_KEY in header_values: - _validate_test_name(name_value=header_values[_TEST_NAME_HEADER_KEY], file_path=file_path) - if _TEST_MODE_HEADER_KEY in header_values: - _validate_test_mode(mode_value=header_values[_TEST_MODE_HEADER_KEY], file_path=file_path) - if SQL_ANALYSIS_CONFIG_KEY in header_values and not isinstance( - header_values[SQL_ANALYSIS_CONFIG_KEY], bool - ): - raise SqlTestParseError(f"TEST() sql_analysis in '{file_path}' must be a boolean") - if not sql_body: - raise SqlTestParseError(f"SQL test '{file_path}' must define SQL after TEST(...)") - - name_value: object | None = header_values.get("name") - test_name: str | None = cast(str | None, name_value) - mode_value: object | None = header_values.get(_TEST_MODE_HEADER_KEY) - test_mode: SqlTestMode = ( - _infer_sql_test_mode(sql_body=sql_body, file_path=file_path) - if mode_value is None - else SqlTestMode(str(mode_value)) - ) - parameters, cases = _parse_test_parameters_and_cases( - header_values=header_values, - file_path=file_path, - ) - cursor_start: str | None = _parse_test_cursor_bound( - header_values=header_values, key=_TEST_CURSOR_START_HEADER_KEY, file_path=file_path - ) - cursor_end: str | None = _parse_test_cursor_bound( - header_values=header_values, key=_TEST_CURSOR_END_HEADER_KEY, file_path=file_path - ) - if (cursor_start is not None or cursor_end is not None) and test_mode != SqlTestMode.MODEL: - raise SqlTestParseError( - f"TEST() in '{file_path}' declares `cursor_start` or `cursor_end`, which are only " - f"supported for model tests, not mode '{test_mode.value}'" - ) - return DiscoveredSqlTestBlock( - test_index=test_index, - header_values=header_values, - sql_body=sql_body, - name=test_name, - mode=test_mode, - parameters=parameters, - cases=cases, - cursor_start=cursor_start, - cursor_end=cursor_end, - sql_body_span=sql_body_span, - ) - - -def _parse_test_cursor_bound( - *, header_values: dict[str, object], key: str, file_path: Path -) -> str | None: - value: object | None = header_values.get(key) - if value is None: - return None - if isinstance(value, datetime | date): - return value.isoformat() - if isinstance(value, int) and not isinstance(value, bool): - return str(value) - if isinstance(value, str) and value.strip(): - return value.strip() - raise SqlTestParseError( - f"TEST() {key} in '{file_path}' must be a non-empty string or an integer" - ) - - -def _parse_test_parameters_and_cases( - *, header_values: dict[str, object], file_path: Path -) -> tuple[tuple[SqlTestParameterDeclaration, ...], tuple[DiscoveredSqlTestCase, ...]]: - raw_parameters: object | None = header_values.get(_TEST_PARAMETERS_HEADER_KEY) - raw_cases: object | None = header_values.get(_TEST_CASES_HEADER_KEY) - if raw_parameters is None and raw_cases is None: - return (), () - if raw_parameters is None or raw_cases is None: - raise SqlTestParseError( - f"TEST() in '{file_path}' must define `parameters` and `cases` together" - ) - if not isinstance(raw_parameters, dict) or not raw_parameters: - raise SqlTestParseError(f"TEST() parameters in '{file_path}' must be a non-empty mapping") - if not isinstance(raw_cases, dict) or not raw_cases: - raise SqlTestParseError(f"TEST() cases in '{file_path}' must be a non-empty mapping") - - parameters: tuple[SqlTestParameterDeclaration, ...] = tuple( - _parse_parameter_declaration( - name=name, - raw_declaration=raw_declaration, - file_path=file_path, - ) - for name, raw_declaration in cast(dict[object, object], raw_parameters).items() - ) - parameter_names: tuple[str, ...] = tuple(parameter.name for parameter in parameters) - cases: tuple[DiscoveredSqlTestCase, ...] = tuple( - _parse_test_case( - name=name, - raw_values=raw_values, - case_index=case_index, - parameters=parameters, - parameter_names=parameter_names, - file_path=file_path, - ) - for case_index, (name, raw_values) in enumerate( - cast(dict[object, object], raw_cases).items() - ) - ) - return parameters, cases - - -def _parse_parameter_declaration( - *, name: object, raw_declaration: object, file_path: Path -) -> SqlTestParameterDeclaration: - parameter_name: str = _parse_test_identifier(value=name, label="parameter", file_path=file_path) - nullable: bool = False - raw_type: object = raw_declaration - if isinstance(raw_declaration, dict): - declaration_options: dict[object, object] = cast(dict[object, object], raw_declaration) - unknown_options: set[object] = set(declaration_options) - {"type", "nullable"} - if unknown_options: - names: str = ", ".join(sorted(str(option) for option in unknown_options)) - raise SqlTestParseError( - f"TEST() parameter '{parameter_name}' in '{file_path}' has unsupported " - f"options: {names}" - ) - raw_type = declaration_options.get("type") - raw_nullable: object = declaration_options.get("nullable", False) - if not isinstance(raw_nullable, bool): - raise SqlTestParseError( - f"TEST() parameter '{parameter_name}' nullable option in '{file_path}' " - "must be boolean" - ) - nullable = raw_nullable - if not isinstance(raw_type, str): - raise SqlTestParseError( - f"TEST() parameter '{parameter_name}' type in '{file_path}' must be an identifier" - ) - try: - value_type: SqlValueKind = SqlValueKind(raw_type.lower()) - except ValueError as error: - raise SqlTestParseError( - f"TEST() parameter '{parameter_name}' in '{file_path}' has unsupported " - f"type '{raw_type}'" - ) from error - if value_type not in _PARAMETER_TYPES: - allowed: str = ", ".join(value.value for value in _PARAMETER_TYPES) - raise SqlTestParseError( - f"TEST() parameter '{parameter_name}' in '{file_path}' must use a scalar " - f"type: {allowed}" - ) - return SqlTestParameterDeclaration( - name=parameter_name, - value_type=value_type, - nullable=nullable, - ) - - -def _parse_test_case( - *, - name: object, - raw_values: object, - case_index: int, - parameters: tuple[SqlTestParameterDeclaration, ...], - parameter_names: tuple[str, ...], - file_path: Path, -) -> DiscoveredSqlTestCase: - case_name: str = _parse_test_identifier(value=name, label="case", file_path=file_path) - if not isinstance(raw_values, dict): - raise SqlTestParseError( - f"TEST() case '{case_name}' in '{file_path}' must define a parameter mapping" - ) - values: dict[object, object] = cast(dict[object, object], raw_values) - missing: tuple[str, ...] = tuple(name for name in parameter_names if name not in values) - extra: tuple[str, ...] = tuple(str(name) for name in values if name not in parameter_names) - if missing: - raise SqlTestParseError( - f"TEST() case '{case_name}' in '{file_path}' is missing parameters: " - f"{', '.join(missing)}" - ) - if extra: - raise SqlTestParseError( - f"TEST() case '{case_name}' in '{file_path}' has undeclared parameters: " - f"{', '.join(extra)}" - ) - normalized_values: list[tuple[str, SqlValue]] = [] - parameter: SqlTestParameterDeclaration - for parameter in parameters: - raw_value: object = values[parameter.name] - if raw_value is None and not parameter.nullable: - raise SqlTestParseError( - f"TEST() case '{case_name}' parameter '{parameter.name}' in '{file_path}' " - "is not nullable" - ) - try: - value: SqlValue = normalize_sql_value( - raw_value=raw_value, - explicit_type=None if raw_value is None else parameter.value_type.value, - context=f"TEST() case '{case_name}' parameter '{parameter.name}' in '{file_path}'", - ) - except SqlValueValidationError as error: - raise SqlTestParseError(str(error)) from error - normalized_values.append((parameter.name, value)) - return DiscoveredSqlTestCase( - name=case_name, - values=tuple(normalized_values), - case_index=case_index, - ) - - -def _parse_test_identifier(*, value: object, label: str, file_path: Path) -> str: - if not isinstance(value, str) or not _IDENTIFIER_PATTERN.fullmatch(value): - raise SqlTestParseError(f"TEST() {label} name in '{file_path}' must be a SQL identifier") - return value - - -def _validate_test_name(*, name_value: object, file_path: Path) -> None: - if not isinstance(name_value, str) or not name_value.strip(): - raise SqlTestParseError(f"TEST() name in '{file_path}' must be a non-empty string") - - -def _validate_test_mode(*, mode_value: object, file_path: Path) -> None: - if not isinstance(mode_value, str): - raise SqlTestParseError(f"TEST() mode in '{file_path}' must be a string") - - allowed_modes: str = ", ".join(mode.value for mode in SqlTestMode) - if mode_value not in {mode.value for mode in SqlTestMode}: - raise SqlTestParseError(f"TEST() mode in '{file_path}' must be one of: {allowed_modes}") - - -def validate_sql_test_names(*, file_path: Path, blocks: tuple[DiscoveredSqlTestBlock, ...]) -> None: - """Reject multi-block files whose blocks lack unique names.""" - - if len(blocks) <= 1: - return - - unnamed_indexes: tuple[int, ...] = tuple( - block.test_index for block in blocks if block.name is None - ) - if unnamed_indexes: - missing_indexes: str = ", ".join(str(index) for index in unnamed_indexes) - raise SqlTestParseError( - f"SQL test '{file_path}' contains multiple TEST blocks; every block must define " - f"a unique `name`. Missing names for blocks: {missing_indexes}" - ) - - seen_names: set[str] = set() - block: DiscoveredSqlTestBlock - for block in blocks: - if block.name is None: - raise SqlTestParseError( - f"SQL test '{file_path}' contains multiple TEST blocks; every block must define " - "a unique `name`" - ) - if block.name in seen_names: - raise SqlTestParseError( - f"SQL test '{file_path}' defines duplicate TEST() name '{block.name}'" - ) - seen_names.add(block.name) diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/supported_keys.py b/src/sqlbuild/compiler/discovery/_helpers/validation/supported_keys.py index fc8dc71052..a9e0d30386 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/supported_keys.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/supported_keys.py @@ -3,32 +3,11 @@ from __future__ import annotations from collections.abc import Mapping -from difflib import get_close_matches from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.discovery.exceptions import DiscoveryError -_SUGGESTION_CUTOFF: float = 0.6 -_LISTED_SUPPORTED_KEYS_LIMIT: int = 12 - - -def unsupported_keys_help(*, keys: tuple[str, ...], supported_keys: frozenset[str]) -> str: - """Suggest the nearest supported key for each unsupported key.""" - - candidates: list[str] = sorted(supported_keys) - suggestions: list[str] = [] - for key in keys: - matches: list[str] = get_close_matches(key, candidates, n=1, cutoff=_SUGGESTION_CUTOFF) - if matches: - suggestions.append( - f"'{matches[0]}'" if len(keys) == 1 else f"'{matches[0]}' for '{key}'" - ) - if suggestions: - return f"did you mean {', '.join(suggestions)}?" - if len(candidates) <= _LISTED_SUPPORTED_KEYS_LIMIT: - return f"supported keys: {', '.join(candidates)}" - return "remove the key; see the reference for supported keys" - def reject_unknown_mapping_keys( *, @@ -40,10 +19,8 @@ def reject_unknown_mapping_keys( ) -> None: """Raise when a YAML mapping declares keys outside its allowed set.""" - unknown: tuple[str, ...] = tuple(sorted(str(key) for key in mapping if key not in allowed)) - if unknown: - raise error_class( - f"{file_path} {label} has unknown keys: {', '.join(unknown)}; allowed keys: " - f"{', '.join(sorted(allowed))}", - help=unsupported_keys_help(keys=unknown, supported_keys=allowed), - ) + failure: tuple[str, str] | None = _native.unknown_mapping_keys( + [(str(key), isinstance(key, str)) for key in mapping], list(allowed) + ) + if failure is not None: + raise error_class(f"{file_path} {label} {failure[0]}", help=failure[1]) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/entry_lines.py b/src/sqlbuild/compiler/discovery/_helpers/yml/entry_lines.py deleted file mode 100644 index f92a845f91..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/entry_lines.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Index the outermost `- name:` entries of authored YAML declaration files.""" - -from __future__ import annotations - -import re -from collections.abc import Iterator, Mapping -from functools import lru_cache - -_ENTRY_PREFIX: re.Pattern[str] = re.compile(r"(?P\s*)-\s*name:") -_COMMENT_START: str = "#" -_QUOTES: tuple[str, ...] = ("", "'", '"') - - -@lru_cache(maxsize=16) -def yaml_entry_lines(contents: str) -> Mapping[str, int]: - """Index every entry name once so per-resource lookups stay linear in file size.""" - - best: dict[str, tuple[int, int]] = {} - index: int - line: str - for index, line in enumerate(contents.splitlines(), start=1): - prefix: re.Match[str] | None = _ENTRY_PREFIX.match(line) - if prefix is None: - continue - indent: int = len(prefix.group("indent")) - for name in _entry_names(line[prefix.end() :]): - current: tuple[int, int] | None = best.get(name) - if current is None or indent < current[0]: - best[name] = (indent, index) - return {name: line_number for name, (_, line_number) in best.items()} - - -def _entry_names(value: str) -> Iterator[str]: - """Yield every name `\\s*(['"]?)\\1\\s*(#.*)?$` accepts for one entry value.""" - - closings: tuple[int, ...] = _closing_positions(value) - for start in range(_leading_whitespace(text=value, start=0) + 1): - for quote in _QUOTES: - if not value.startswith(quote, start): - continue - body_start: int = start + len(quote) - for closing in closings: - end: int = closing - len(quote) - if end >= body_start and value.startswith(quote, end): - yield value[body_start:end] - - -def _closing_positions(value: str) -> tuple[int, ...]: - """Return positions after which only whitespace and an optional comment remain.""" - - positions: list[int] = [] - tail: int = len(value) - while tail >= 0: - position: int = tail - positions.append(position) - while position > 0 and value[position - 1].isspace(): - position -= 1 - positions.append(position) - tail = value.rfind(_COMMENT_START, 0, tail) - return tuple(sorted(set(positions))) - - -def _leading_whitespace(*, text: str, start: int) -> int: - position: int = start - while position < len(text) and text[position].isspace(): - position += 1 - return position diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index 82cdc324bc..b885047716 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -2,455 +2,27 @@ from __future__ import annotations -import tomllib -from dataclasses import fields -from datetime import date, datetime -from decimal import Decimal -from enum import StrEnum from pathlib import Path from typing import cast -import yaml -from yaml import YAMLError - -from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.authored_values.main._change_policy_problem import change_policy_problem -from sqlbuild.compiler.authored_values.main._change_policy_toml_help import ( - change_policy_toml_help, -) -from sqlbuild.compiler.compile.constants import MAX_MICROBATCHES_CONFIG_KEY, TEMPLATE_OPEN_TOKEN -from sqlbuild.compiler.discovery._helpers.validation.supported_keys import unsupported_keys_help -from sqlbuild.compiler.discovery._helpers.yml.target_diff import load_target_diff +import sqlbuild._native as _native from sqlbuild.compiler.discovery._helpers.yml.target_warehouses import ( - load_target_warehouses, reject_unsupported_target_warehouses, ) -from sqlbuild.compiler.discovery.constants import ( - CONFIG_CONCURRENCY_KEY, - DBT_DEFER_CLONE_CONFIG_KEY, - DBT_LEGACY_REUSE_FROM_CONFIG_KEY, - DBT_PRODUCTION_REF_CONFIG_KEY, - DBT_REPLAY_ON_CHANGE_CONFIG_KEY, - LEGACY_CONFIG_CONCURRENCY_KEY, - LEGACY_LOCAL_CONFIG_FILENAME, - LEGACY_PROJECT_CONFIG_FILENAME, - LEGACY_SQL_VALIDATION_CONFIG_KEY, - LOCAL_CONFIG_FILENAME, - MODELS_DIRECTORY_NAME, - PROJECT_CONFIG_FILENAME, - QUERY_CHANGE_TRACKING_SETTING_KEY, - REQUIRE_SQL_ANALYSIS_SETTING_KEY, - SETTINGS_SECTION, - SQL_ANALYSIS_CONFIG_KEY, - SQL_MODEL_HEADER_KEYS, - TOML_FILE_SUFFIX, -) -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.compiler.path_defaults.constants import GLOB_SEGMENTS, UNSUPPORTED_GLOB_MARKERS -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.cost.constants import USD_PER_CREDIT_CONFIG_KEY -from sqlbuild.cursor_algebra.constants import DURATION_DAY_UNIT -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.errors.setting_help.main.join_helps import join_helps -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.errors.setting_help.main.setting_note import setting_note -from sqlbuild.spec.contracts.constants import ( - SCENARIO_RUN_NAMESPACE_CONFIG_KEY, - TIME_TRAVEL_RETENTION_MATERIALIZATIONS, - ZERO_DAY_CURSOR_DURATION, -) -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.parse_scenario_run_namespace import parse_scenario_run_namespace -from sqlbuild.spec.contracts.models import ( - AuthoredTimeTravelRetention, - ClonePolicy, - ConstantsConfig, - CostConfig, - CursorsConfig, - DbtConfig, - DefaultsConfig, - DiffConfig, - ExecutionLimitsConfig, - FutureCursorsConfig, - JanitorConfig, - LifecycleEventSinkFilterConfig, - LifecycleEventSinksConfig, - LocalClonePolicy, - LocalConfig, - LocalDbtConfig, - LocalTargetConfig, - MaterializationDefaultsConfig, - MaterializationRetentionDefaults, - MicrobatchesConfig, - MicrobatchLimitsConfig, - MigrationsConfig, - ProjectConfig, - ReferencesConfig, - ScenarioConfig, - ScenarioSnapshotLimitsConfig, - ScopesConfig, - SettingsConfig, - SinksConfig, - SnapshotsConfig, - StartCursorsConfig, - TargetConfig, -) -from sqlbuild.spec.contracts.types import ( - ColumnContractMode, - EventExportSeverity, - FutureCursorAction, - MicrobatchLimitAction, - MissingMigrationOriginPolicy, - RetentionDecreasePolicy, - TableType, - TableTypeDowngradePolicy, - TableTypeValue, - TimeTravelRetentionValue, -) -from sqlbuild.sql_values.types import CollectionRendering - -_SNAPSHOT_FULL_REFRESH_POLICIES: frozenset[str] = frozenset( - {"allow", "deny", "require_confirmation"} -) -_SNAPSHOT_SCHEMA_CHANGE_POLICIES: frozenset[str] = frozenset( - {"append_new_columns", "deny", "require_confirmation"} -) -_DEFAULT_CURRENT_STATE_SNAPSHOT_FULL_REFRESH: str = "deny" -_DEFAULT_HISTORICAL_SNAPSHOT_FULL_REFRESH: str = "require_confirmation" -_DEFAULT_SNAPSHOT_SCHEMA_CHANGE: str = "append_new_columns" -_DEFAULT_WILDCARD_CHECK_SNAPSHOT_SCHEMA_CHANGE: str = "require_confirmation" -_BATCH_CONCURRENCY_CONFIG_KEY: str = "batch_concurrency" -_MAX_BATCHES_CONFIG_KEY: str = "max_batches" -_EVENT_EXPORTER_FILTER_KEYS: frozenset[str] = frozenset({"event_kinds", "min_severity"}) -_DEFAULTS_KEYS: frozenset[str] = frozenset( - { - "append_cursor_inclusive", - "batch_concurrency", - "batch_size", - "contract", - "cursor_end", - "cursor_future_action", - "cursor_future_max_distance", - "cursor_start", - "cursor_start_max_action", - "cursor_start_max_ahead", - "cursor_watermark_mode", - "database", - "full_refresh", - "function_database", - "function_schema", - "incremental_mode", - "incremental_strategy", - "lookback", - "materialized", - "max_microbatches", - "merge_exclude_columns", - "microbatch_strategy", - "replay_on_change", - "row_diff_exclude_columns", - "row_diff_sample_rows", - "row_diff_sample_seed", - "row_diff_tolerances", - "schema", - "seed_database", - "seed_schema", - "tags", - "unaccounted_partition_policy", - } -) -_LIFECYCLE_SHUTDOWN_TIMEOUT_CONFIG_KEY: str = "shutdown_timeout" -_LIFECYCLE_SHUTDOWN_TIMEOUT_MAX_SECONDS: int = 600 -_ZERO_FIXED_DURATION: str = "0s" -_EVENT_EXPORTER_KEYS: frozenset[str] = _EVENT_EXPORTER_FILTER_KEYS | frozenset( - {"named", _LIFECYCLE_SHUTDOWN_TIMEOUT_CONFIG_KEY} -) -_LEGACY_EVENT_EXPORTERS_CONFIG_KEY: str = "event_exporters" -_OLD_NAME_VIEWS_CONFIG_KEY: str = "old_name_views" +from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig def load_project_config(*, project_dir: Path) -> ProjectConfig: """Load project config from the given project directory.""" - file_path: Path = _resolve_project_config_path(project_dir=project_dir) - payload: dict[str, object] = _load_config_mapping(file_path=file_path) - if _LEGACY_EVENT_EXPORTERS_CONFIG_KEY in payload: - raise ProjectConfigError(f"{file_path} event_exporters was replaced by sinks.lifecycle") - - name: str = _require_str(payload=payload, key="name", file_path=file_path) - adapter: str = _require_str(payload=payload, key="adapter", file_path=file_path) - default_target: str | None = _optional_str(payload=payload, key="default_target") - connection: dict[str, object] = _optional_mapping(payload=payload, key="connection") - connections: dict[str, dict[str, object]] = _load_connections( - payload=payload.get("connections"), file_path=file_path - ) - settings: SettingsConfig = _load_settings(payload=payload.get("settings"), file_path=file_path) - scopes: ScopesConfig = _load_scopes(payload=payload.get("scopes"), file_path=file_path) - references: ReferencesConfig = _load_references( - payload=payload.get("references"), file_path=file_path - ) - cost: CostConfig = _load_cost(payload=payload.get("cost"), file_path=file_path) - constants: ConstantsConfig = _load_constants( - payload=payload.get("constants"), file_path=file_path - ) - cursors: CursorsConfig = _load_cursors(payload=payload.get("cursors"), file_path=file_path) - microbatches: MicrobatchesConfig = _load_microbatches( - payload=payload.get("microbatches"), file_path=file_path - ) - defaults: DefaultsConfig = _load_defaults(payload=payload.get("defaults"), file_path=file_path) - diff: DiffConfig = _load_diff(payload=payload.get("diff"), file_path=file_path) - materialization_defaults: MaterializationDefaultsConfig = _load_materialization_defaults( - payload=payload.get("materialization_defaults"), file_path=file_path - ) - path_defaults: dict[str, dict[str, object]] = _load_path_defaults( - payload=payload.get("path_defaults"), - file_path=file_path, - ) - vars_map: dict[str, str] = _load_string_mapping( - payload=payload.get("vars"), file_path=file_path - ) - targets: dict[str, TargetConfig] = _load_targets( - payload=payload.get("targets"), - file_path=file_path, - ) - janitor: JanitorConfig = _load_janitor(payload=payload.get("janitor"), file_path=file_path) - migrations: MigrationsConfig = _load_migrations( - payload=payload.get("migrations"), file_path=file_path - ) - snapshots: SnapshotsConfig = _load_snapshots( - payload=payload.get("snapshots"), file_path=file_path - ) - scenario: ScenarioConfig = _load_scenario(payload=payload.get("scenario"), file_path=file_path) - dbt: DbtConfig = _load_dbt(payload=payload.get("dbt"), file_path=file_path) - sinks: SinksConfig = _load_sinks(payload=payload.get("sinks"), file_path=file_path) - if janitor.enabled and janitor.delete_tracked_only and not settings.query_change_tracking: - raise ProjectConfigError( - f"{file_path} [janitor] delete_tracked_only = true requires query change tracking; " - + setting_note( - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=QUERY_CHANGE_TRACKING_SETTING_KEY, - value=False, - ), - help=setting_help( - purpose="to track which relations SQLBuild created", - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=QUERY_CHANGE_TRACKING_SETTING_KEY, - value=True, - ), - ) - - return ProjectConfig( - name=name, - adapter=adapter, - default_target=default_target, - connection=connection, - connections=connections, - settings=settings, - scopes=scopes, - references=references, - cost=cost, - constants=constants, - cursors=cursors, - microbatches=microbatches, - defaults=defaults, - diff=diff, - materialization_defaults=materialization_defaults, - path_defaults=path_defaults, - vars=vars_map, - targets=targets, - janitor=janitor, - migrations=migrations, - snapshots=snapshots, - scenario=scenario, - dbt=dbt, - sinks=sinks, - ) - - -def _load_diff(*, payload: object, file_path: Path) -> DiffConfig: - mapping: dict[str, object] = _coerce_mapping(payload=payload, label="diff", file_path=file_path) - key: str = "query_artifact_ttl" - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({key}), - label="diff", - file_path=file_path, - ) - value: str = _optional_str(payload=mapping, key=key) or "24h" - if _fixed_duration_seconds(value=value) is None: - raise ProjectConfigError( - f"{file_path} diff.{key} must be a positive fixed duration such as '24h'" - ) - return DiffConfig(query_artifact_ttl=value) - - -def _load_sinks(*, payload: object, file_path: Path) -> SinksConfig: - if payload is None: - return SinksConfig() - if not isinstance(payload, dict): - raise ProjectConfigError(f"{file_path} sinks must be a mapping") - mapping: dict[str, object] = cast(dict[str, object], payload) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({"lifecycle"}), - label="sinks", - file_path=file_path, - ) - return SinksConfig( - lifecycle=_load_lifecycle_sinks( - payload=mapping.get("lifecycle"), - file_path=file_path, - ) - ) - - -def _load_lifecycle_sinks(*, payload: object, file_path: Path) -> LifecycleEventSinksConfig: - from sqlbuild.runtime.event_exporting.constants import ( - EVENT_EXPORT_KINDS, - EVENT_EXPORT_SEVERITIES, - ) - - if payload is None: - return LifecycleEventSinksConfig() - if not isinstance(payload, dict): - raise ProjectConfigError(f"{file_path} sinks.lifecycle must be a mapping") - config_mapping: dict[str, object] = cast(dict[str, object], payload) - _validate_allowed_keys( - mapping=config_mapping, - allowed_keys=_EVENT_EXPORTER_KEYS, - label="sinks.lifecycle", - file_path=file_path, - ) - - def load_filter(*, value: object, label: str) -> LifecycleEventSinkFilterConfig: - if not isinstance(value, dict): - raise ProjectConfigError(f"{file_path} {label} must be a mapping") - filter_mapping: dict[str, object] = cast(dict[str, object], value) - _validate_allowed_keys( - mapping=filter_mapping, - allowed_keys=_EVENT_EXPORTER_FILTER_KEYS, - label=label, - file_path=file_path, - ) - kinds_value: object = filter_mapping.get("event_kinds") - kinds: frozenset[str] | None = None - if kinds_value is not None: - if not isinstance(kinds_value, list) or not all( - isinstance(kind, str) for kind in kinds_value - ): - raise ProjectConfigError( - f"{file_path} {label}.event_kinds must be a list of strings" - ) - kinds = frozenset(cast(list[str], kinds_value)) - unknown: frozenset[str] = kinds - EVENT_EXPORT_KINDS - if unknown: - raise ProjectConfigError( - f"{file_path} {label}.event_kinds contains unknown kind(s): " - + ", ".join(sorted(unknown)) - ) - severity_value: object = filter_mapping.get("min_severity") - severity: EventExportSeverity | None = None - if severity_value is not None: - if not isinstance(severity_value, str) or severity_value not in EVENT_EXPORT_SEVERITIES: - raise ProjectConfigError( - f"{file_path} {label}.min_severity must be one of: " - + ", ".join(EVENT_EXPORT_SEVERITIES) - ) - severity = EventExportSeverity(severity_value) - return LifecycleEventSinkFilterConfig(event_kinds=kinds, min_severity=severity) - - named_value: object = config_mapping.get("named", {}) - if not isinstance(named_value, dict): - raise ProjectConfigError(f"{file_path} sinks.lifecycle.named must be a mapping") - named: dict[str, LifecycleEventSinkFilterConfig] = {} - for name, value in named_value.items(): - if not isinstance(name, str) or not name: - raise ProjectConfigError(f"{file_path} sinks.lifecycle.named contains an invalid name") - named[name] = load_filter(value=value, label=f"sinks.lifecycle.named.{name}") - defaults_mapping: dict[str, object] = { - key: value for key, value in config_mapping.items() if key in _EVENT_EXPORTER_FILTER_KEYS - } - shutdown_timeout, shutdown_timeout_seconds = _load_lifecycle_shutdown_timeout( - value=config_mapping.get(_LIFECYCLE_SHUTDOWN_TIMEOUT_CONFIG_KEY), - file_path=file_path, - ) - return LifecycleEventSinksConfig( - defaults=load_filter(value=defaults_mapping, label="sinks.lifecycle"), - named=named, - shutdown_timeout=shutdown_timeout, - shutdown_timeout_seconds=shutdown_timeout_seconds, - ) - - -def _load_lifecycle_shutdown_timeout( - *, value: object, file_path: Path -) -> tuple[str | None, int | None]: - if value is None: - return None, None - message: str = ( - f"{file_path} sinks.lifecycle.{_LIFECYCLE_SHUTDOWN_TIMEOUT_CONFIG_KEY} must be a fixed " - f"duration from '0s' to '10m' such as '30s'" - ) - if not isinstance(value, str): - raise ProjectConfigError(message) - seconds: int | None = _fixed_duration_seconds(value=value, allow_zero=True) - if seconds is None or seconds > _LIFECYCLE_SHUTDOWN_TIMEOUT_MAX_SECONDS: - raise ProjectConfigError(message) - return value, seconds - - -def _fixed_duration_seconds(*, value: str, allow_zero: bool = False) -> int | None: - """Return a fixed duration's seconds, or None when it is invalid or calendar-based.""" - - if allow_zero and value == _ZERO_FIXED_DURATION: - return 0 - duration: Duration | None = Duration.parse(value) - if duration is None or duration.has_calendar_component: - return None - return duration.fixed_seconds + return cast(ProjectConfig, _native.load_project_config_file(project_dir)) def load_local_config(*, project_dir: Path) -> LocalConfig: """Load local config if present.""" - file_path: Path = _resolve_local_config_path(project_dir=project_dir) - if not file_path.exists(): - return LocalConfig() - - payload: dict[str, object] = _load_config_mapping(file_path=file_path) - target: str | None = _optional_str(payload=payload, key="target") - adapter: str | None = _optional_str(payload=payload, key="adapter") - connection: dict[str, object] = _optional_mapping(payload=payload, key="connection") - connections: dict[str, dict[str, object]] = _load_connections( - payload=payload.get("connections"), file_path=file_path - ) - targets: dict[str, LocalTargetConfig] = _load_local_targets( - payload=payload.get("targets"), - file_path=file_path, - ) - local_settings_result: tuple[SettingsConfig, frozenset[str]] = _load_local_settings( - payload=payload.get("settings"), file_path=file_path - ) - settings: SettingsConfig = local_settings_result[0] - setting_overrides: frozenset[str] = local_settings_result[1] - vars_map: dict[str, str] = _load_string_mapping( - payload=payload.get("vars"), file_path=file_path - ) - dbt: LocalDbtConfig = _load_local_dbt(payload=payload.get("dbt"), file_path=file_path) - scenario: ScenarioConfig = _load_scenario(payload=payload.get("scenario"), file_path=file_path) - return LocalConfig( - target=target, - adapter=adapter, - connection=connection, - connections=connections, - targets=targets, - settings=settings, - setting_overrides=setting_overrides, - vars=vars_map, - dbt=dbt, - scenario=scenario, - ) + return cast(LocalConfig, _native.load_local_config_file(project_dir)) def validate_local_sql_analysis_policy( @@ -458,31 +30,12 @@ def validate_local_sql_analysis_policy( ) -> None: """Reject a local project-wide SQL analysis opt-out when the project requires analysis.""" - if ( - not project_config.settings.require_sql_analysis - or SQL_ANALYSIS_CONFIG_KEY not in local_config.setting_overrides - or local_config.settings.sql_analysis - ): - return - raise ProjectConfigError( - f"{_resolve_local_config_path(project_dir=project_dir)} turns SQL analysis off with " - f"[settings] {SQL_ANALYSIS_CONFIG_KEY} = false, which this project does not allow; " - + setting_note( - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, - value=True, - ), - help=join_helps( - f"remove {SQL_ANALYSIS_CONFIG_KEY} from [settings] in {LOCAL_CONFIG_FILENAME}; " - "to skip SQL analysis for one run, use `--no-sql-analysis`", - setting_help( - purpose="to allow turning SQL analysis off locally", - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, - value=False, - ), + _native.validate_local_sql_analysis( + project_dir, + ( + project_config.settings.require_sql_analysis, + SQL_ANALYSIS_CONFIG_KEY in local_config.setting_overrides, + local_config.settings.sql_analysis, ), ) @@ -496,1501 +49,6 @@ def validate_target_warehouses_adapter( project_dir=project_dir, project_config=project_config, local_config=local_config, - project_config_path=_resolve_project_config_path(project_dir=project_dir), - local_config_path=_resolve_local_config_path(project_dir=project_dir), - ) - - -def _resolve_project_config_path(*, project_dir: Path) -> Path: - toml_path: Path = project_dir / PROJECT_CONFIG_FILENAME - if toml_path.exists(): - return toml_path - legacy_path: Path = project_dir / LEGACY_PROJECT_CONFIG_FILENAME - if legacy_path.exists(): - raise ProjectConfigError( - f"{legacy_path} is no longer supported. Rename it to {PROJECT_CONFIG_FILENAME} " - "and convert the contents to TOML." - ) - return toml_path - - -def _resolve_local_config_path(*, project_dir: Path) -> Path: - toml_path: Path = project_dir / LOCAL_CONFIG_FILENAME - if toml_path.exists(): - return toml_path - yaml_path: Path = project_dir / LEGACY_LOCAL_CONFIG_FILENAME - if yaml_path.exists(): - raise ProjectConfigError( - f"{yaml_path} is no longer supported. Rename it to {LOCAL_CONFIG_FILENAME} " - "and convert the contents to TOML." - ) - return toml_path - - -def _load_config_mapping(*, file_path: Path) -> dict[str, object]: - if file_path.suffix == TOML_FILE_SUFFIX: - return _load_toml_mapping(file_path=file_path) - return _load_yaml_mapping(file_path=file_path) - - -def _load_toml_mapping(*, file_path: Path) -> dict[str, object]: - if not file_path.exists(): - raise ProjectConfigError( - f"Project config not found: {file_path}. Run sqb from a sqlbuild project " - "directory or pass --project-dir." - ) - try: - with file_path.open("rb") as config_file: - payload: object = tomllib.load(config_file) - except tomllib.TOMLDecodeError as error: - raise ProjectConfigError(f"{file_path} contains invalid TOML: {error}") from error - if not isinstance(payload, dict): - raise ProjectConfigError(f"{file_path} must contain a top-level mapping") - return cast(dict[str, object], payload) - - -def _load_yaml_mapping(*, file_path: Path) -> dict[str, object]: - if not file_path.exists(): - raise ProjectConfigError( - f"Project config not found: {file_path}. Run sqb from a sqlbuild project " - "directory or pass --project-dir." - ) - contents: str = file_path.read_text(encoding="utf-8") - try: - payload: object = yaml.safe_load(contents) - except YAMLError as error: - raise ProjectConfigError(f"{file_path} contains invalid YAML: {error}") from error - if payload is None: - return {} - if not isinstance(payload, dict): - raise ProjectConfigError(f"{file_path} must contain a top-level mapping") - return cast(dict[str, object], payload) - - -def _require_str(*, payload: dict[str, object], key: str, file_path: Path) -> str: - value: object | None = payload.get(key) - if not isinstance(value, str) or not value.strip(): - raise ProjectConfigError(f"{file_path} must define non-empty string '{key}'") - return value.strip() - - -def _optional_str(*, payload: dict[str, object], key: str) -> str | None: - value: object | None = payload.get(key) - if value is None: - return None - if not isinstance(value, str) or not value.strip(): - raise ProjectConfigError(f"Expected '{key}' to be a non-empty string when provided") - return value.strip() - - -def _optional_mapping(*, payload: dict[str, object], key: str) -> dict[str, object]: - value: object | None = payload.get(key) - if value is None: - return {} - if not isinstance(value, dict): - raise ProjectConfigError(f"Expected '{key}' to be a mapping when provided") - return cast(dict[str, object], value) - - -def _load_settings(*, payload: object, file_path: Path) -> SettingsConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="settings", file_path=file_path - ) - _reject_removed_virtual_keys( - mapping=mapping, - removed_keys=frozenset({"virtual_environments", "changes_only"}), - label="settings", - file_path=file_path, - ) - canonical_setting_names: frozenset[str] = frozenset( - field.name for field in fields(SettingsConfig) - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=canonical_setting_names - | {LEGACY_CONFIG_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_CONFIG_KEY}, - label="settings", - file_path=file_path, - ) - if CONFIG_CONCURRENCY_KEY in mapping and LEGACY_CONFIG_CONCURRENCY_KEY in mapping: - raise ProjectConfigError( - "settings cannot define both 'concurrency' and legacy 'max_concurrency'" - ) - if ( - SQL_ANALYSIS_CONFIG_KEY in mapping - and LEGACY_SQL_VALIDATION_CONFIG_KEY in mapping - and mapping[SQL_ANALYSIS_CONFIG_KEY] != mapping[LEGACY_SQL_VALIDATION_CONFIG_KEY] - ): - raise ProjectConfigError( - "settings.sql_analysis conflicts with legacy settings.sql_validation" - ) - analysis_key: str = ( - SQL_ANALYSIS_CONFIG_KEY - if SQL_ANALYSIS_CONFIG_KEY in mapping - else LEGACY_SQL_VALIDATION_CONFIG_KEY - ) - sql_analysis: bool = _optional_bool(mapping=mapping, key=analysis_key, default=True) - require_sql_analysis: bool = _optional_bool( - mapping=mapping, key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, default=False - ) - query_change_tracking: bool = _optional_bool( - mapping=mapping, - key="query_change_tracking", - default=True, - ) - raw_column_contract_mode: object = mapping.get( - "column_contract_mode", ColumnContractMode.IMPLICIT.value - ) - try: - column_contract_mode: ColumnContractMode = ColumnContractMode(raw_column_contract_mode) - except (TypeError, ValueError) as error: - raise ProjectConfigError( - "settings.column_contract_mode must be one of: implicit, explicit" - ) from error - auto_load_sources: bool = _optional_bool(mapping=mapping, key="auto_load_sources", default=True) - microbatch_concurrency: bool = _optional_bool( - mapping=mapping, key="microbatch_concurrency", default=False - ) - microbatch_unaccounted_partition_policy: str = ( - _optional_str(payload=mapping, key="microbatch_unaccounted_partition_policy") - or "synthesize" - ) - valid_unaccounted_policies: frozenset[str] = frozenset( - {"synthesize", "recover_empty", "recover_all"} - ) - if microbatch_unaccounted_partition_policy not in valid_unaccounted_policies: - raise ProjectConfigError( - "settings.microbatch_unaccounted_partition_policy must be one of: " - + ", ".join(sorted(valid_unaccounted_policies)) - ) - concurrency_key: str = ( - LEGACY_CONFIG_CONCURRENCY_KEY - if LEGACY_CONFIG_CONCURRENCY_KEY in mapping - else CONFIG_CONCURRENCY_KEY - ) - concurrency: int = _optional_int(mapping=mapping, key=concurrency_key, default=1) - if concurrency < 1: - raise ProjectConfigError("settings.concurrency must be >= 1") - table_promotion_mode: str | None = _optional_str(payload=mapping, key="table_promotion_mode") - raw_default_audit_severity: str | None = _optional_str( - payload=mapping, key="default_audit_severity" - ) - default_audit_severity: AuditSeverity | None = None - if raw_default_audit_severity is not None: - try: - default_audit_severity = AuditSeverity(raw_default_audit_severity) - except ValueError as error: - allowed: str = ", ".join(item.value for item in AuditSeverity) - raise ProjectConfigError( - f"settings.default_audit_severity must be one of: {allowed}" - ) from error - default_audit_run_scope: str | None = _optional_str( - payload=mapping, key="default_audit_run_scope" - ) - return SettingsConfig( - sql_analysis=sql_analysis, - require_sql_analysis=require_sql_analysis, - query_change_tracking=query_change_tracking, - column_contract_mode=column_contract_mode, - concurrency=concurrency, - auto_load_sources=auto_load_sources, - microbatch_concurrency=microbatch_concurrency, - microbatch_unaccounted_partition_policy=microbatch_unaccounted_partition_policy, - table_promotion_mode=table_promotion_mode, - default_audit_severity=default_audit_severity, - default_audit_run_scope=default_audit_run_scope, - ) - - -def _load_cursors(*, payload: object, file_path: Path) -> CursorsConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="cursors", file_path=file_path - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({"future", "start"}), - label="cursors", - file_path=file_path, - ) - future: dict[str, object] = _coerce_mapping( - payload=mapping.get("future"), label="cursors.future", file_path=file_path - ) - _validate_allowed_keys( - mapping=future, - allowed_keys=frozenset({"max_distance", "action"}), - label="cursors.future", - file_path=file_path, - ) - max_distance: str | None = _optional_str(payload=future, key="max_distance") - if ( - max_distance is not None - and max_distance != ZERO_DAY_CURSOR_DURATION - and Duration.parse(max_distance) is None - ): - raise ProjectConfigError( - "cursors.future.max_distance must be a duration like 0d, 7d, 12h, or 1mo" - ) - raw_action: str | None = _optional_str(payload=future, key="action") - try: - action: FutureCursorAction = FutureCursorAction(raw_action or FutureCursorAction.ERROR) - except ValueError: - raise ProjectConfigError("cursors.future.action must be one of: cap, error") from None - start: dict[str, object] = _coerce_mapping( - payload=mapping.get("start"), label="cursors.start", file_path=file_path - ) - _validate_allowed_keys( - mapping=start, - allowed_keys=frozenset({"max_ahead", "action"}), - label="cursors.start", - file_path=file_path, - ) - max_ahead: str | None = _optional_str(payload=start, key="max_ahead") - if ( - max_ahead is not None - and max_ahead != ZERO_DAY_CURSOR_DURATION - and Duration.parse(max_ahead) is None - ): - raise ProjectConfigError( - "cursors.start.max_ahead must be a duration like 0d, 7d, 12h, or 1mo" - ) - raw_start_action: str | None = _optional_str(payload=start, key="action") - try: - start_action: FutureCursorAction = FutureCursorAction( - raw_start_action or FutureCursorAction.ERROR - ) - except ValueError: - raise ProjectConfigError("cursors.start.action must be one of: cap, error") from None - return CursorsConfig( - future=FutureCursorsConfig(max_distance=max_distance, action=action), - start=StartCursorsConfig(max_ahead=max_ahead, action=start_action), - ) - - -def _load_microbatches(*, payload: object, file_path: Path) -> MicrobatchesConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="microbatches", file_path=file_path - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({"limits"}), - label="microbatches", - file_path=file_path, - ) - limits: dict[str, object] = _coerce_mapping( - payload=mapping.get("limits"), label="microbatches.limits", file_path=file_path - ) - _validate_allowed_keys( - mapping=limits, - allowed_keys=frozenset({_MAX_BATCHES_CONFIG_KEY, "action"}), - label="microbatches.limits", - file_path=file_path, - ) - max_batches: int | None = None - if _MAX_BATCHES_CONFIG_KEY in limits: - if isinstance(limits[_MAX_BATCHES_CONFIG_KEY], bool): - raise ProjectConfigError("microbatches.limits.max_batches must be a positive integer") - max_batches = _optional_int(mapping=limits, key=_MAX_BATCHES_CONFIG_KEY, default=0) - if max_batches < 1: - raise ProjectConfigError("microbatches.limits.max_batches must be a positive integer") - raw_action: str = _optional_str(payload=limits, key="action") or MicrobatchLimitAction.ERROR - try: - action: MicrobatchLimitAction = MicrobatchLimitAction(raw_action) - except ValueError: - raise ProjectConfigError("microbatches.limits.action must be one of: error, warn") from None - if action not in {MicrobatchLimitAction.ERROR, MicrobatchLimitAction.WARN}: - raise ProjectConfigError("microbatches.limits.action must be one of: error, warn") - return MicrobatchesConfig(limits=MicrobatchLimitsConfig(max_batches=max_batches, action=action)) - - -def _load_migrations(*, payload: object, file_path: Path) -> MigrationsConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="migrations", file_path=file_path - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({_OLD_NAME_VIEWS_CONFIG_KEY}), - label="migrations", - file_path=file_path, - ) - if _OLD_NAME_VIEWS_CONFIG_KEY not in mapping: - return MigrationsConfig() - value: object = mapping[_OLD_NAME_VIEWS_CONFIG_KEY] - if value is False: - return MigrationsConfig(old_name_views=None) - if isinstance(value, str) and Duration.parse(value.strip()) is not None: - return MigrationsConfig(old_name_views=value.strip()) - raise ProjectConfigError( - f"{file_path} migrations.old_name_views must be a positive duration such as '30d', " - "or false to disable compatibility views at old model names" - ) - - -def _load_scopes(*, payload: object, file_path: Path) -> ScopesConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="scopes", file_path=file_path - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({"enforce_placement"}), - label="scopes", - file_path=file_path, - ) - return ScopesConfig( - enforce_placement=_optional_bool(mapping=mapping, key="enforce_placement", default=True) - ) - - -def _load_references(*, payload: object, file_path: Path) -> ReferencesConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="references", file_path=file_path - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({"enforce_explicit"}), - label="references", - file_path=file_path, - ) - return ReferencesConfig( - enforce_explicit=_optional_bool(mapping=mapping, key="enforce_explicit", default=True) - ) - - -def _load_cost(*, payload: object, file_path: Path) -> CostConfig: - mapping: dict[str, object] = _coerce_mapping(payload=payload, label="cost", file_path=file_path) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({USD_PER_CREDIT_CONFIG_KEY}), - label="cost", - file_path=file_path, - ) - if USD_PER_CREDIT_CONFIG_KEY not in mapping: - return CostConfig() - raw_value: object = mapping[USD_PER_CREDIT_CONFIG_KEY] - if isinstance(raw_value, bool) or not isinstance(raw_value, int | float): - raise ProjectConfigError(f"{file_path} cost.usd_per_credit must be a number") - value: Decimal = Decimal(str(raw_value)) - if not value.is_finite() or value <= 0: - raise ProjectConfigError( - f"{file_path} cost.usd_per_credit must be a finite number greater than zero" - ) - return CostConfig(usd_per_credit=value, usd_per_credit_is_default=False) - - -def _load_constants(*, payload: object, file_path: Path) -> ConstantsConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="constants", file_path=file_path - ) - key: str = "collection_rendering" - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({key}), - label="constants", - file_path=file_path, - ) - raw_value: object = mapping.get(key, CollectionRendering.VALUE_LIST.value) - if not isinstance(raw_value, str): - raise ProjectConfigError(f"{file_path} constants.{key} must be a string") - try: - collection_rendering: CollectionRendering = CollectionRendering(raw_value) - except ValueError as error: - valid_values: str = ", ".join(rendering.value for rendering in CollectionRendering) - raise ProjectConfigError( - f"{file_path} constants.{key} must be one of: {valid_values}" - ) from error - return ConstantsConfig(collection_rendering=collection_rendering) - - -def _load_local_settings( - *, payload: object, file_path: Path -) -> tuple[SettingsConfig, frozenset[str]]: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="settings", file_path=file_path - ) - if REQUIRE_SQL_ANALYSIS_SETTING_KEY in mapping: - raise ProjectConfigError( - f"{file_path} cannot set [settings] {REQUIRE_SQL_ANALYSIS_SETTING_KEY}; " - f"it is a shared project policy that only {PROJECT_CONFIG_FILENAME} can set", - help=join_helps( - f"remove {REQUIRE_SQL_ANALYSIS_SETTING_KEY} from {LOCAL_CONFIG_FILENAME}", - setting_help( - purpose="to change the policy for everyone", - file_name=PROJECT_CONFIG_FILENAME, - section=SETTINGS_SECTION, - key=REQUIRE_SQL_ANALYSIS_SETTING_KEY, - value=not bool(mapping[REQUIRE_SQL_ANALYSIS_SETTING_KEY]), - ), - ), - ) - setting_names: frozenset[str] = frozenset(field.name for field in fields(SettingsConfig)) - normalized_overrides: set[str] = set() - key: str - for key in mapping: - if key == LEGACY_CONFIG_CONCURRENCY_KEY: - normalized_overrides.add(CONFIG_CONCURRENCY_KEY) - elif key == LEGACY_SQL_VALIDATION_CONFIG_KEY: - normalized_overrides.add(SQL_ANALYSIS_CONFIG_KEY) - elif key in setting_names: - normalized_overrides.add(key) - return ( - _load_settings(payload=payload, file_path=file_path), - frozenset(normalized_overrides), - ) - - -def _validate_allowed_keys( - *, - mapping: dict[str, object], - allowed_keys: frozenset[str], - label: str, - file_path: Path, -) -> None: - unknown_keys: tuple[str, ...] = tuple(sorted(key for key in mapping if key not in allowed_keys)) - if not unknown_keys: - return - allowed: str = ", ".join(sorted(allowed_keys)) - unknown: str = ", ".join(unknown_keys) - raise ProjectConfigError( - f"{file_path} {label} contains unknown key(s): {unknown}. Allowed keys: {allowed}", - help=unsupported_keys_help(keys=unknown_keys, supported_keys=allowed_keys), - ) - - -def _validate_change_policies(*, mapping: dict[str, object], section: str, file_path: Path) -> None: - key: str - for key in ("on_schema_change", "replay_on_change"): - value: object | None = mapping.get(key) - if value is None or (isinstance(value, str) and TEMPLATE_OPEN_TOKEN in value): - continue - problem: str | None = change_policy_problem(key=key, value=value) - if problem is not None: - raise ProjectConfigError( - f"{file_path} [{section}] {problem}", - help=change_policy_toml_help(key=key, file_name=file_path.name, section=section), - ) - - -def _normalize_path_default_key(*, path_key: str, file_path: Path) -> str: - normalized_key: str = path_key.strip() - if not normalized_key: - raise ProjectConfigError(f"{file_path} path_defaults contains an empty path key") - if normalized_key.startswith("/"): - raise ProjectConfigError( - f"{file_path} path_defaults['{path_key}'] is invalid. Use model-relative paths " - "without a leading slash, for example 'staging' or 'staging/nested'." - ) - if normalized_key.endswith("/"): - raise ProjectConfigError( - f"{file_path} path_defaults['{path_key}'] is invalid. Remove the trailing slash and " - "use a model-relative path such as 'staging' or 'staging/nested'." - ) - path_parts: list[str] = normalized_key.split("/") - if any(not part for part in path_parts): - raise ProjectConfigError( - f"{file_path} path_defaults['{path_key}'] is invalid. Path defaults cannot contain " - "empty path segments." - ) - if path_parts[0] == MODELS_DIRECTORY_NAME: - raise ProjectConfigError( - f"{file_path} path_defaults['{path_key}'] uses redundant 'models/' prefix. " - "Use a model-relative path such as 'staging' or 'staging/nested'." - ) - unsupported_glob_parts_list: list[str] = [] - for part in path_parts: - if part in GLOB_SEGMENTS: - continue - if any(marker in part for marker in UNSUPPORTED_GLOB_MARKERS): - unsupported_glob_parts_list.append(part) - unsupported_glob_parts: tuple[str, ...] = tuple(unsupported_glob_parts_list) - if unsupported_glob_parts: - unsupported: str = ", ".join(unsupported_glob_parts) - raise ProjectConfigError( - f"{file_path} path_defaults['{path_key}'] contains unsupported glob segment(s): " - f"{unsupported}. Use '*' or '**' as complete path segments." - ) - return "/".join(path_parts) - - -def _load_defaults(*, payload: object, file_path: Path) -> DefaultsConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="defaults", file_path=file_path - ) - _reject_removed_virtual_keys( - mapping=mapping, - removed_keys=frozenset({"run_despite_unchanged"}), - label="defaults", - file_path=file_path, - ) - _validate_allowed_keys( - mapping=mapping, allowed_keys=_DEFAULTS_KEYS, label="[defaults]", file_path=file_path - ) - _validate_change_policies(mapping=mapping, section="defaults", file_path=file_path) - row_diff_exclude_columns: tuple[str, ...] = tuple( - _load_string_sequence( - payload=mapping.get("row_diff_exclude_columns"), - label="defaults.row_diff_exclude_columns", - file_path=file_path, - ) - ) - merge_exclude_columns: tuple[str, ...] = tuple( - _load_string_sequence( - payload=mapping.get("merge_exclude_columns"), - label="defaults.merge_exclude_columns", - file_path=file_path, - ) - ) - tags: tuple[str, ...] = tuple( - _load_string_sequence( - payload=mapping.get("tags"), - label="defaults.tags", - file_path=file_path, - ) - ) - row_diff_tolerances: dict[str, object] = _optional_mapping( - payload=mapping, - key="row_diff_tolerances", - ) - return DefaultsConfig( - materialized=_optional_str(payload=mapping, key="materialized"), - database=_optional_str(payload=mapping, key="database"), - schema=_optional_str(payload=mapping, key="schema"), - seed_database=_optional_str(payload=mapping, key="seed_database"), - seed_schema=_optional_str(payload=mapping, key="seed_schema"), - function_database=_optional_str(payload=mapping, key="function_database"), - function_schema=_optional_str(payload=mapping, key="function_schema"), - contract=_optional_contract_policy(mapping=mapping, key="contract"), - incremental_strategy=_optional_str(payload=mapping, key="incremental_strategy"), - incremental_mode=_optional_str(payload=mapping, key="incremental_mode"), - microbatch_strategy=_optional_str(payload=mapping, key="microbatch_strategy"), - cursor_watermark_mode=_optional_str(payload=mapping, key="cursor_watermark_mode"), - merge_exclude_columns=merge_exclude_columns, - full_refresh=_optional_nullable_bool(mapping=mapping, key="full_refresh"), - append_cursor_inclusive=_optional_templated_bool( - mapping=mapping, - key="append_cursor_inclusive", - ), - cursor_start=_optional_cursor_start(mapping=mapping, key="cursor_start"), - cursor_end=_optional_cursor_start(mapping=mapping, key="cursor_end"), - cursor_start_max_ahead=_optional_str(payload=mapping, key="cursor_start_max_ahead"), - cursor_start_max_action=_optional_str(payload=mapping, key="cursor_start_max_action"), - cursor_future_max_distance=_optional_str(payload=mapping, key="cursor_future_max_distance"), - cursor_future_action=_optional_str(payload=mapping, key="cursor_future_action"), - lookback=_optional_str(payload=mapping, key="lookback"), - batch_size=_optional_scalar_batch_size(mapping=mapping, key="batch_size"), - batch_concurrency=( - _optional_int(mapping=mapping, key=_BATCH_CONCURRENCY_CONFIG_KEY, default=1) - if _BATCH_CONCURRENCY_CONFIG_KEY in mapping - else None - ), - max_microbatches=( - _optional_int(mapping=mapping, key=MAX_MICROBATCHES_CONFIG_KEY, default=1) - if MAX_MICROBATCHES_CONFIG_KEY in mapping - else None - ), - unaccounted_partition_policy=_optional_str( - payload=mapping, key="unaccounted_partition_policy" - ), - replay_on_change=_optional_str(payload=mapping, key="replay_on_change"), - row_diff_exclude_columns=row_diff_exclude_columns, - row_diff_tolerances=row_diff_tolerances, - row_diff_sample_rows=_optional_non_negative_int( - mapping=mapping, key="row_diff_sample_rows" - ), - row_diff_sample_seed=_optional_strict_int(mapping=mapping, key="row_diff_sample_seed"), - tags=tags, - ) - - -def _load_path_defaults(*, payload: object, file_path: Path) -> dict[str, dict[str, object]]: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="path_defaults", file_path=file_path - ) - path_defaults: dict[str, dict[str, object]] = {} - path_key: str - path_value: object - for path_key, path_value in mapping.items(): - if not isinstance(path_value, dict): - raise ProjectConfigError(f"{file_path} path_defaults['{path_key}'] must be a mapping") - path_dict: dict[str, object] = cast(dict[str, object], path_value) - _reject_removed_virtual_keys( - mapping=path_dict, - removed_keys=frozenset({"run_despite_unchanged"}), - label="path_defaults", - file_path=file_path, - ) - _validate_allowed_keys( - mapping=path_dict, - allowed_keys=SQL_MODEL_HEADER_KEYS, - label=f"path_defaults['{path_key}']", - file_path=file_path, - ) - _validate_change_policies( - mapping=path_dict, - section=f'path_defaults."{path_key}"', - file_path=file_path, - ) - _validate_path_default_tags(path_dict=path_dict, path_key=path_key, file_path=file_path) - normalized_path_key: str = _normalize_path_default_key( - path_key=path_key, file_path=file_path - ) - if normalized_path_key in path_defaults: - raise ProjectConfigError( - f"{file_path} path_defaults defines both '{path_key}' and another key that " - f"normalize to '{normalized_path_key}'. Use only the canonical model-relative " - "form without a leading 'models/' prefix." - ) - path_defaults[normalized_path_key] = path_dict - return path_defaults - - -def _load_materialization_defaults( - *, payload: object, file_path: Path -) -> MaterializationDefaultsConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="materialization_defaults", file_path=file_path - ) - materializations: tuple[str, ...] = TIME_TRAVEL_RETENTION_MATERIALIZATIONS - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset(materializations), - label="materialization_defaults", - file_path=file_path, - ) - loaded: dict[str, MaterializationRetentionDefaults] = {} - for materialization in materializations: - materialization_mapping: dict[str, object] = _coerce_mapping( - payload=mapping.get(materialization), - label=f"materialization_defaults.{materialization}", - file_path=file_path, - ) - _validate_allowed_keys( - mapping=materialization_mapping, - allowed_keys=frozenset({"time_travel_retention", "table_type"}), - label=f"materialization_defaults.{materialization}", - file_path=file_path, - ) - loaded[materialization] = MaterializationRetentionDefaults( - time_travel_retention=_optional_retention_policy( - mapping=materialization_mapping, - key="time_travel_retention", - label=f"materialization_defaults.{materialization}.time_travel_retention", - file_path=file_path, - allow_inherit=True, - ), - table_type=_optional_table_type( - mapping=materialization_mapping, - key="table_type", - label=f"materialization_defaults.{materialization}.table_type", - file_path=file_path, - allow_inherit=True, - ), - ) - return MaterializationDefaultsConfig(**loaded) - - -def _optional_retention_policy( - *, - mapping: dict[str, object], - key: str, - label: str, - file_path: Path, - allow_inherit: bool = False, -) -> AuthoredTimeTravelRetention | None: - value: object | None = mapping.get(key) - if value is None: - return None - if not isinstance(value, str): - raise ProjectConfigError(f"{file_path} {label} must be a whole-day string like '7d'") - if allow_inherit and value == TimeTravelRetentionValue.INHERIT: - return None - if value == TimeTravelRetentionValue.DISABLED: - return AuthoredTimeTravelRetention(unmanaged=True) - if value == ZERO_DAY_CURSOR_DURATION: - return AuthoredTimeTravelRetention(desired_days=0) - duration: Duration | None = Duration.parse(value) - if duration is None or duration.units != frozenset({DURATION_DAY_UNIT}): - allowed_keywords: str = ", 'inherit', or 'disabled'" if allow_inherit else " or 'disabled'" - raise ProjectConfigError( - f"{file_path} {label} must be a whole-day string like '7d'{allowed_keywords}" - ) - return AuthoredTimeTravelRetention(desired_days=duration.days) - - -def _optional_table_type( - *, - mapping: dict[str, object], - key: str, - label: str, - file_path: Path, - allow_inherit: bool = False, -) -> TableType | None: - value: object | None = mapping.get(key) - if value is None or allow_inherit and value == TableTypeValue.INHERIT: - return None - try: - return TableType(value) - except (TypeError, ValueError) as exc: - suffix: str = ", or 'inherit'" if allow_inherit else "" - raise ProjectConfigError( - f"{file_path} {label} must be 'permanent' or 'transient'{suffix}" - ) from exc - - -def _optional_target_policy[PolicyT: StrEnum]( - *, - mapping: dict[str, object], - key: str, - target_name: str, - file_path: Path, - policy: type[PolicyT], -) -> PolicyT | None: - value: object | None = mapping.get(key) - if value is None: - return None - try: - return policy(value) - except (TypeError, ValueError) as exc: - raise ProjectConfigError( - f"{file_path} targets.{target_name}.{key} must be 'deny', " - "'require_confirmation', or 'allow'" - ) from exc - - -def _optional_target_retention_default( - *, mapping: dict[str, object], target_name: str, file_path: Path -) -> AuthoredTimeTravelRetention | None: - if isinstance(mapping.get("time_travel_retention"), dict): - return None - return _optional_retention_policy( - mapping=mapping, - key="time_travel_retention", - label=f"targets.{target_name}.time_travel_retention", - file_path=file_path, - ) - - -def _target_retention_by_materialization( - *, mapping: dict[str, object], target_name: str, file_path: Path -) -> dict[str, AuthoredTimeTravelRetention]: - payload: object | None = mapping.get("time_travel_retention") - if not isinstance(payload, dict): - return {} - label: str = f"targets.{target_name}.time_travel_retention" - retention_mapping: dict[str, object] = _coerce_mapping( - payload=payload, label=label, file_path=file_path - ) - _validate_allowed_keys( - mapping=retention_mapping, - allowed_keys=frozenset(TIME_TRAVEL_RETENTION_MATERIALIZATIONS), - label=label, - file_path=file_path, - ) - loaded: dict[str, AuthoredTimeTravelRetention] = {} - for materialization in TIME_TRAVEL_RETENTION_MATERIALIZATIONS: - policy: AuthoredTimeTravelRetention | None = _optional_retention_policy( - mapping=retention_mapping, - key=materialization, - label=f"{label}.{materialization}", - file_path=file_path, - ) - if policy is not None: - loaded[materialization] = policy - return loaded - - -def _load_targets(*, payload: object, file_path: Path) -> dict[str, TargetConfig]: - return { - name: _project_target(local) - for name, local in _load_local_targets(payload=payload, file_path=file_path).items() - } - - -def _project_target(local: LocalTargetConfig) -> TargetConfig: - defaults: TargetConfig = TargetConfig() - return TargetConfig( - connection=local.connection, - connection_name=local.connection_name, - vars=local.vars, - database=local.database, - schema=local.schema, - loader_schema=local.loader_schema, - defer_sources_to=local.defer_sources_to, - defer_clone_from=local.defer_clone_from, - clone=ClonePolicy( - allow_as_clone_origin=local.clone.allow_as_clone_origin is True, - allow_as_clone_destination=local.clone.allow_as_clone_destination is True, - ), - compile_cache=local.compile_cache, - time_travel_retention=local.time_travel_retention, - time_travel_retention_by_materialization=( - local.time_travel_retention_by_materialization or {} - ), - owns_time_travel_retention_namespace=local.owns_time_travel_retention_namespace is True, - default_table_type=local.default_table_type, - table_type_downgrade=local.table_type_downgrade or defaults.table_type_downgrade, - time_travel_retention_decrease=( - local.time_travel_retention_decrease or defaults.time_travel_retention_decrease - ), - missing_migration_origin=local.missing_migration_origin - or defaults.missing_migration_origin, - execution_limits=local.execution_limits, - diff=local.diff, - warehouses=local.warehouses, - ) - - -def _load_local_targets(*, payload: object, file_path: Path) -> dict[str, LocalTargetConfig]: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="targets", file_path=file_path - ) - targets: dict[str, LocalTargetConfig] = {} - target_name: str - target_payload: object - for target_name, target_payload in mapping.items(): - target_mapping: dict[str, object] = _coerce_mapping( - payload=target_payload, - label=f"targets.{target_name}", - file_path=file_path, - ) - _validate_target_keys( - target_mapping=target_mapping, target_name=target_name, file_path=file_path - ) - clone_mapping: dict[str, object] = _coerce_mapping( - payload=target_mapping.get("clone"), - label=f"targets.{target_name}.clone", - file_path=file_path, - ) - _validate_clone_keys( - clone_mapping=clone_mapping, target_name=target_name, file_path=file_path - ) - execution_limits: ExecutionLimitsConfig = _load_execution_limits( - payload=target_mapping.get("execution_limits"), - label=f"targets.{target_name}.execution_limits", - file_path=file_path, - ) - connection, connection_name = _load_target_connection( - target_mapping=target_mapping, target_name=target_name, file_path=file_path - ) - targets[target_name] = LocalTargetConfig( - connection=connection, - connection_name=connection_name, - vars=_load_string_mapping(payload=target_mapping.get("vars"), file_path=file_path), - database=_optional_str(payload=target_mapping, key="database"), - schema=_optional_str(payload=target_mapping, key="schema"), - loader_schema=_optional_str(payload=target_mapping, key="loader_schema"), - defer_sources_to=_optional_str(payload=target_mapping, key="defer_sources_to"), - defer_clone_from=_optional_str(payload=target_mapping, key="defer_clone_from"), - compile_cache=_optional_nullable_bool(mapping=target_mapping, key="compile_cache"), - time_travel_retention=_optional_target_retention_default( - mapping=target_mapping, target_name=target_name, file_path=file_path - ), - time_travel_retention_by_materialization=( - _target_retention_by_materialization( - mapping=target_mapping, target_name=target_name, file_path=file_path - ) - if isinstance(target_mapping.get("time_travel_retention"), dict) - else None - ), - owns_time_travel_retention_namespace=_optional_nullable_bool( - mapping=target_mapping, - key="owns_time_travel_retention_namespace", - ), - default_table_type=_optional_table_type( - mapping=target_mapping, - key="default_table_type", - label=f"targets.{target_name}.default_table_type", - file_path=file_path, - ), - table_type_downgrade=_optional_target_policy( - mapping=target_mapping, - key="table_type_downgrade", - target_name=target_name, - file_path=file_path, - policy=TableTypeDowngradePolicy, - ), - time_travel_retention_decrease=_optional_target_policy( - mapping=target_mapping, - key="time_travel_retention_decrease", - target_name=target_name, - file_path=file_path, - policy=RetentionDecreasePolicy, - ), - missing_migration_origin=_optional_target_policy( - mapping=target_mapping, - key="missing_migration_origin", - target_name=target_name, - file_path=file_path, - policy=MissingMigrationOriginPolicy, - ), - execution_limits=execution_limits, - diff=load_target_diff( - payload=target_mapping.get("diff"), target_name=target_name, file_path=file_path - ), - warehouses=load_target_warehouses( - payload=target_mapping.get("warehouses"), - target_name=target_name, - file_path=file_path, - ), - clone=LocalClonePolicy( - allow_as_clone_origin=_optional_nullable_bool( - mapping=clone_mapping, - key="allow_as_clone_origin", - ), - allow_as_clone_destination=_optional_nullable_bool( - mapping=clone_mapping, - key="allow_as_clone_destination", - ), - ), - ) - return targets - - -def _load_connections(*, payload: object, file_path: Path) -> dict[str, dict[str, object]]: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="connections", file_path=file_path - ) - connections: dict[str, dict[str, object]] = {} - for name, value in mapping.items(): - if not name.strip(): - raise ProjectConfigError(f"{file_path} connections contains an empty name") - connections[name] = dict( - _coerce_mapping(payload=value, label=f"connections.{name}", file_path=file_path) - ) - return connections - - -def _load_target_connection( - *, target_mapping: dict[str, object], target_name: str, file_path: Path -) -> tuple[dict[str, object], str | None]: - value: object | None = target_mapping.get("connection") - if value is None: - return {}, None - if isinstance(value, dict): - return cast(dict[str, object], value), None - if isinstance(value, str) and value.strip(): - return {}, value.strip() - raise ProjectConfigError( - f"{file_path} targets.{target_name}.connection must be a non-empty string or mapping" - ) - - -def _validate_target_keys( - *, target_mapping: dict[str, object], target_name: str, file_path: Path -) -> None: - _reject_removed_virtual_keys( - mapping=target_mapping, - removed_keys=frozenset({"state", "changes_only"}), - label=f"targets.{target_name}", - file_path=file_path, - ) - _validate_allowed_keys( - mapping=target_mapping, - allowed_keys=frozenset( - { - "connection", - "vars", - "database", - "schema", - "loader_schema", - "defer_sources_to", - "defer_clone_from", - "compile_cache", - "time_travel_retention", - "owns_time_travel_retention_namespace", - "default_table_type", - "table_type_downgrade", - "time_travel_retention_decrease", - "missing_migration_origin", - "clone", - "execution_limits", - "diff", - "warehouses", - } - ), - label=f"targets.{target_name}", - file_path=file_path, - ) - - -def _load_execution_limits( - *, payload: object, label: str, file_path: Path -) -> ExecutionLimitsConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, - label=label, - file_path=file_path, - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({"max_models", "max_duration", "remediation"}), - label=label, - file_path=file_path, - ) - max_models: int | None = _optional_strict_int(mapping=mapping, key="max_models") - if max_models is not None and max_models < 1: - raise ProjectConfigError(f"{file_path} {label}.max_models must be >= 1") - max_duration: str | None = _optional_str(payload=mapping, key="max_duration") - max_duration_seconds: int | None = None - if max_duration is not None: - max_duration_seconds = _fixed_duration_seconds(value=max_duration) - if max_duration_seconds is None: - raise ProjectConfigError( - f"{file_path} {label}.max_duration must be a positive fixed duration such as '30m'" - ) - return ExecutionLimitsConfig( - max_models=max_models, - max_duration=max_duration, - max_duration_seconds=max_duration_seconds, - remediation=_optional_str(payload=mapping, key="remediation"), - ) - - -def _validate_clone_keys( - *, clone_mapping: dict[str, object], target_name: str, file_path: Path -) -> None: - _validate_allowed_keys( - mapping=clone_mapping, - allowed_keys=frozenset({"allow_as_clone_origin", "allow_as_clone_destination"}), - label=f"targets.{target_name}.clone", - file_path=file_path, - ) - - -def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="janitor", file_path=file_path - ) - _reject_removed_virtual_keys( - mapping=mapping, - removed_keys=frozenset({"max_checkpoints"}), - label="janitor", - file_path=file_path, - ) - enabled: bool = _optional_bool(mapping=mapping, key="enabled", default=False) - retention_days: int | None = _optional_nullable_int(mapping=mapping, key="retention_days") - archive_retention_days: int = _optional_int( - mapping=mapping, key="archive_retention_days", default=14 - ) - direct_state_history_versions: int = _optional_int( - mapping=mapping, - key="direct_state_history_versions", - default=20, - ) - delete_tracked_only: bool = _optional_bool( - mapping=mapping, - key="delete_tracked_only", - default=True, - ) - exclude_patterns: tuple[str, ...] = tuple( - _load_string_sequence( - payload=mapping.get("exclude_patterns"), - label="janitor.exclude_patterns", - file_path=file_path, - ) - ) - if retention_days is not None and retention_days < 0: - raise ProjectConfigError(f"{file_path} janitor.retention_days must be >= 0") - if archive_retention_days < 0: - raise ProjectConfigError(f"{file_path} janitor.archive_retention_days must be >= 0") - if direct_state_history_versions < 0: - raise ProjectConfigError(f"{file_path} janitor.direct_state_history_versions must be >= 0") - return JanitorConfig( - enabled=enabled, - retention_days=retention_days, - archive_retention_days=archive_retention_days, - direct_state_history_versions=direct_state_history_versions, - delete_tracked_only=delete_tracked_only, - exclude_patterns=exclude_patterns, - ) - - -def _reject_removed_virtual_keys( - *, - mapping: dict[str, object], - removed_keys: frozenset[str], - label: str, - file_path: Path, -) -> None: - removed: list[str] = sorted(removed_keys.intersection(mapping)) - if removed: - raise ProjectConfigError( - f"{file_path} [{label}] option(s) {', '.join(removed)} were removed " - "with virtual environments; projects run in direct mode" - ) - - -def _load_snapshots(*, payload: object, file_path: Path) -> SnapshotsConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="snapshots", file_path=file_path - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset( - { - "current_state_full_refresh", - "historical_full_refresh", - "schema_change", - "wildcard_check_schema_change", - } - ), - label="snapshots", - file_path=file_path, + project_config_path=_native.resolve_config_file_path(project_dir, False), + local_config_path=_native.resolve_config_file_path(project_dir, True), ) - return SnapshotsConfig( - current_state_full_refresh=_optional_full_refresh_policy( - mapping=mapping, - key="current_state_full_refresh", - default=_DEFAULT_CURRENT_STATE_SNAPSHOT_FULL_REFRESH, - ), - historical_full_refresh=_optional_full_refresh_policy( - mapping=mapping, - key="historical_full_refresh", - default=_DEFAULT_HISTORICAL_SNAPSHOT_FULL_REFRESH, - ), - schema_change=_optional_snapshot_schema_change_policy( - mapping=mapping, - key="schema_change", - default=_DEFAULT_SNAPSHOT_SCHEMA_CHANGE, - ), - wildcard_check_schema_change=_optional_snapshot_schema_change_policy( - mapping=mapping, - key="wildcard_check_schema_change", - default=_DEFAULT_WILDCARD_CHECK_SNAPSHOT_SCHEMA_CHANGE, - ), - ) - - -def _load_scenario(*, payload: object, file_path: Path) -> ScenarioConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="scenario", file_path=file_path - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset( - {"local_type_overrides", "snapshot_limits", SCENARIO_RUN_NAMESPACE_CONFIG_KEY} - ), - label="scenario", - file_path=file_path, - ) - overrides_payload: dict[str, object] = _coerce_mapping( - payload=mapping.get("local_type_overrides"), - label="scenario.local_type_overrides", - file_path=file_path, - ) - local_type_overrides: dict[str, dict[str, str]] = {} - dialect: str - rules_payload: object - for dialect, rules_payload in overrides_payload.items(): - if not isinstance(dialect, str) or not dialect.strip(): - raise ProjectConfigError( - f"{file_path} scenario.local_type_overrides contains an empty dialect key" - ) - local_type_overrides[dialect.strip()] = _load_string_mapping( - payload=rules_payload, - file_path=file_path, - ) - run_namespace: str | None = None - if SCENARIO_RUN_NAMESPACE_CONFIG_KEY in mapping: - try: - run_namespace = parse_scenario_run_namespace(mapping[SCENARIO_RUN_NAMESPACE_CONFIG_KEY]) - except SpecConfigError as exc: - raise ProjectConfigError(f"{file_path} {exc}") from exc - return ScenarioConfig( - run_namespace=run_namespace, - local_type_overrides=local_type_overrides, - snapshot_limits=_load_scenario_snapshot_limits( - payload=mapping.get("snapshot_limits"), - file_path=file_path, - ), - ) - - -def _load_dbt(*, payload: object, file_path: Path) -> DbtConfig: - """Load optional dbt interop configuration.""" - - mapping: dict[str, object] = _coerce_mapping(payload=payload, label="dbt", file_path=file_path) - reuse_keys: tuple[str, ...] = tuple( - key - for key in (DBT_LEGACY_REUSE_FROM_CONFIG_KEY, DBT_PRODUCTION_REF_CONFIG_KEY) - if key in mapping - ) - if reuse_keys: - raise ProjectConfigError( - f"{file_path} [dbt] reuse option(s) were removed: {', '.join(reuse_keys)}; " - "dbt production-ref reuse is no longer supported" - ) - removed_keys: tuple[str, ...] = tuple( - key - for key in (DBT_DEFER_CLONE_CONFIG_KEY, DBT_REPLAY_ON_CHANGE_CONFIG_KEY) - if key in mapping - ) - if removed_keys: - raise ProjectConfigError( - f"{file_path} [dbt] option(s) were removed: {', '.join(removed_keys)}; " - "use dbt-native --state/--defer" - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset( - { - "project_dir", - "profiles_dir", - "target", - "target_path", - "vars", - } - ), - label="dbt", - file_path=file_path, - ) - return DbtConfig( - project_dir=_optional_str(payload=mapping, key="project_dir"), - profiles_dir=_optional_str(payload=mapping, key="profiles_dir"), - target=_optional_str(payload=mapping, key="target"), - target_path=_optional_str(payload=mapping, key="target_path"), - vars=_load_object_mapping(payload=mapping.get("vars"), file_path=file_path), - ) - - -def _load_local_dbt(*, payload: object, file_path: Path) -> LocalDbtConfig: - """Load optional local dbt interop configuration.""" - - mapping: dict[str, object] = _coerce_mapping(payload=payload, label="dbt", file_path=file_path) - if DBT_DEFER_CLONE_CONFIG_KEY in mapping: - raise ProjectConfigError( - f"{file_path} [dbt] defer_clone_from was removed; " - "remove it and use dbt-native --state/--defer" - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset({"target", "vars"}), - label="dbt", - file_path=file_path, - ) - return LocalDbtConfig( - target=_optional_str(payload=mapping, key="target"), - vars=_load_object_mapping(payload=mapping.get("vars"), file_path=file_path), - ) - - -def _load_scenario_snapshot_limits( - *, payload: object, file_path: Path -) -> ScenarioSnapshotLimitsConfig: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, - label="scenario.snapshot_limits", - file_path=file_path, - ) - _validate_allowed_keys( - mapping=mapping, - allowed_keys=frozenset( - { - "max_rows_per_relation", - "max_total_rows", - "max_bytes_per_relation", - "max_total_bytes", - } - ), - label="scenario.snapshot_limits", - file_path=file_path, - ) - limits: ScenarioSnapshotLimitsConfig = ScenarioSnapshotLimitsConfig( - max_rows_per_relation=_optional_nullable_int( - mapping=mapping, - key="max_rows_per_relation", - ), - max_total_rows=_optional_nullable_int(mapping=mapping, key="max_total_rows"), - max_bytes_per_relation=_optional_nullable_int( - mapping=mapping, - key="max_bytes_per_relation", - ), - max_total_bytes=_optional_nullable_int(mapping=mapping, key="max_total_bytes"), - ) - value: int | None - for value in ( - limits.max_rows_per_relation, - limits.max_total_rows, - limits.max_bytes_per_relation, - limits.max_total_bytes, - ): - if value is not None and value < 0: - raise ProjectConfigError(f"{file_path} scenario.snapshot_limits values must be >= 0") - return limits - - -def _load_string_mapping(*, payload: object, file_path: Path) -> dict[str, str]: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="mapping", file_path=file_path - ) - result: dict[str, str] = {} - key: str - value: object - for key, value in mapping.items(): - if not isinstance(value, str): - raise ProjectConfigError(f"{file_path} expected string value for '{key}'") - result[key] = value - return result - - -def _load_object_mapping(*, payload: object, file_path: Path) -> dict[str, object]: - mapping: dict[str, object] = _coerce_mapping( - payload=payload, label="mapping", file_path=file_path - ) - return dict(mapping) - - -def _load_string_sequence(*, payload: object, label: str, file_path: Path) -> list[str]: - if payload is None: - return [] - if not isinstance(payload, list): - raise ProjectConfigError(f"{file_path} {label} must be a list of strings") - values: list[str] = [] - item: object - for item in payload: - if not isinstance(item, str): - raise ProjectConfigError(f"{file_path} {label} must contain only strings") - values.append(item) - return values - - -def _coerce_mapping(*, payload: object, label: str, file_path: Path) -> dict[str, object]: - if payload is None: - return {} - if not isinstance(payload, dict): - raise ProjectConfigError(f"{file_path} {label} must be a mapping") - return cast(dict[str, object], payload) - - -def _optional_bool(*, mapping: dict[str, object], key: str, default: bool) -> bool: - value: object | None = mapping.get(key) - if value is None: - return default - if not isinstance(value, bool): - raise ProjectConfigError(f"Expected '{key}' to be a boolean when provided") - return value - - -def _optional_nullable_bool(*, mapping: dict[str, object], key: str) -> bool | None: - value: object | None = mapping.get(key) - if value is None: - return None - if not isinstance(value, bool): - raise ProjectConfigError(f"Expected '{key}' to be a boolean when provided") - return value - - -def _optional_templated_bool(*, mapping: dict[str, object], key: str) -> object | None: - value: object | None = mapping.get(key) - if value is None: - return None - if not isinstance(value, bool | str): - raise ProjectConfigError(f"Expected '{key}' to be a boolean when provided") - return value - - -def _optional_int(*, mapping: dict[str, object], key: str, default: int) -> int: - value: object | None = mapping.get(key) - if value is None: - return default - if not isinstance(value, int): - raise ProjectConfigError(f"Expected '{key}' to be an integer when provided") - return value - - -def _optional_nullable_int(*, mapping: dict[str, object], key: str) -> int | None: - value: object | None = mapping.get(key) - if value is None: - return None - if not isinstance(value, int): - raise ProjectConfigError(f"Expected '{key}' to be an integer when provided") - return value - - -def _optional_non_negative_int(*, mapping: dict[str, object], key: str) -> int | None: - value: int | None = _optional_strict_int(mapping=mapping, key=key) - if value is not None and value < 0: - raise ProjectConfigError(f"Expected '{key}' to be zero or greater when provided") - return value - - -def _optional_strict_int(*, mapping: dict[str, object], key: str) -> int | None: - if isinstance(mapping.get(key), bool): - raise ProjectConfigError(f"Expected '{key}' to be an integer when provided") - return _optional_nullable_int(mapping=mapping, key=key) - - -def _optional_scalar_batch_size(*, mapping: dict[str, object], key: str) -> str | int | None: - value: object | None = mapping.get(key) - if value is None: - return None - if isinstance(value, (str, int)): - return value - raise ProjectConfigError(f"Expected '{key}' to be a string or integer when provided") - - -def _optional_full_refresh_policy(*, mapping: dict[str, object], key: str, default: str) -> str: - value: object | None = mapping.get(key) - if value is None: - return default - if not isinstance(value, str): - raise ProjectConfigError(f"Expected '{key}' to be a string when provided") - if value not in _SNAPSHOT_FULL_REFRESH_POLICIES: - valid_values: str = ", ".join(sorted(_SNAPSHOT_FULL_REFRESH_POLICIES)) - raise ProjectConfigError(f"Expected '{key}' to be one of: {valid_values}") - return value - - -def _optional_snapshot_schema_change_policy( - *, mapping: dict[str, object], key: str, default: str -) -> str: - value: object | None = mapping.get(key) - if value is None: - return default - if not isinstance(value, str): - raise ProjectConfigError(f"Expected '{key}' to be a string when provided") - if value not in _SNAPSHOT_SCHEMA_CHANGE_POLICIES: - valid_values: str = ", ".join(sorted(_SNAPSHOT_SCHEMA_CHANGE_POLICIES)) - raise ProjectConfigError(f"Expected '{key}' to be one of: {valid_values}") - return value - - -def _optional_contract_policy(*, mapping: dict[str, object], key: str) -> str | None: - value: object | None = mapping.get(key) - if value is None: - return None - if not isinstance(value, str): - raise ProjectConfigError(f"Expected '{key}' to be a string when provided") - if value not in ContractPolicy: - raise ProjectConfigError("Expected 'contract' to be one of: enforced, none") - return value - - -def _optional_cursor_start(*, mapping: dict[str, object], key: str) -> object | None: - value: object | None = mapping.get(key) - if value is None: - return None - if isinstance(value, str | int | date | datetime): - return value - raise ProjectConfigError( - f"Expected '{key}' to be a string, integer, or date-like value when provided" - ) - - -def _validate_path_default_tags( - *, - path_dict: dict[str, object], - path_key: str, - file_path: Path, -) -> None: - """Validate that tags in a path_defaults entry is a list of strings.""" - - raw_tags: object | None = path_dict.get("tags") - if raw_tags is None: - return - if not isinstance(raw_tags, list): - raise ProjectConfigError(f"{file_path} path_defaults['{path_key}'].tags must be a list") - item: object - for item in raw_tags: - if not isinstance(item, str): - raise ProjectConfigError( - f"{file_path} path_defaults['{path_key}'].tags entries must be strings" - ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/target_diff.py b/src/sqlbuild/compiler/discovery/_helpers/yml/target_diff.py deleted file mode 100644 index e85c326c03..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/target_diff.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Loading and validation for per-target diff limits.""" - -from __future__ import annotations - -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.discovery._helpers.validation.supported_keys import ( - reject_unknown_mapping_keys, -) -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.spec.contracts.constants import UNLIMITED_DIFF_FULL_ROWS -from sqlbuild.spec.contracts.models import TargetDiffConfig - -_MAX_FULL_ROWS_KEY: str = "max_full_rows" - - -def load_target_diff(*, payload: object, target_name: str, file_path: Path) -> TargetDiffConfig: - """Parse one target's optional `[targets..diff]` section.""" - - label: str = f"targets.{target_name}.diff" - if payload is None: - return TargetDiffConfig() - if not isinstance(payload, dict): - raise ProjectConfigError(f"{file_path} {label} must be a mapping") - mapping: dict[str, object] = cast(dict[str, object], payload) - reject_unknown_mapping_keys( - mapping=mapping, - allowed=frozenset({_MAX_FULL_ROWS_KEY}), - file_path=file_path, - label=label, - error_class=ProjectConfigError, - ) - value: object | None = mapping.get(_MAX_FULL_ROWS_KEY) - if value is None: - return TargetDiffConfig() - if value == UNLIMITED_DIFF_FULL_ROWS: - return TargetDiffConfig(max_full_rows="unlimited") - if isinstance(value, bool) or not isinstance(value, int) or value < 1: - raise ProjectConfigError( - f"{file_path} {label}.{_MAX_FULL_ROWS_KEY} must be an integer >= 1 or " - f"'{UNLIMITED_DIFF_FULL_ROWS}'" - ) - return TargetDiffConfig(max_full_rows=value) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/target_warehouses.py b/src/sqlbuild/compiler/discovery/_helpers/yml/target_warehouses.py index 63cc5bda42..cdd96a1c37 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/target_warehouses.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/target_warehouses.py @@ -1,16 +1,12 @@ -"""Loading and adapter validation for per-target command-group warehouses.""" +"""Adapter validation for per-target command-group warehouses.""" from __future__ import annotations from pathlib import Path -from typing import cast from sqlbuild.adapter.discovery.main.session_warehouse_support import ( adapter_supports_session_warehouse, ) -from sqlbuild.compiler.discovery._helpers.validation.supported_keys import ( - reject_unknown_mapping_keys, -) from sqlbuild.compiler.discovery.constants import LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.errors.setting_help.main.setting_note import setting_note @@ -21,38 +17,6 @@ from sqlbuild.spec.contracts.types import WarehouseGroup -def load_target_warehouses( - *, payload: object, target_name: str, file_path: Path -) -> TargetWarehousesConfig: - """Parse one target's optional `[targets..warehouses]` section.""" - - label: str = f"targets.{target_name}.warehouses" - if payload is None: - return TargetWarehousesConfig() - if not isinstance(payload, dict): - raise ProjectConfigError(f"{file_path} {label} must be a mapping") - mapping: dict[str, object] = cast(dict[str, object], payload) - reject_unknown_mapping_keys( - mapping=mapping, - allowed=frozenset(group.value for group in WarehouseGroup), - file_path=file_path, - label=label, - error_class=ProjectConfigError, - ) - values: dict[WarehouseGroup, str | None] = {} - group: WarehouseGroup - for group in WarehouseGroup: - value: object | None = mapping.get(group.value) - if value is not None and (not isinstance(value, str) or not value.strip()): - raise ProjectConfigError( - f"{file_path} {label}.{group.value} must be a non-empty warehouse name" - ) - values[group] = value.strip() if isinstance(value, str) else None - return TargetWarehousesConfig( - build=values[WarehouseGroup.BUILD], query=values[WarehouseGroup.QUERY] - ) - - def reject_unsupported_target_warehouses( *, project_dir: Path, diff --git a/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py b/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py index 81e06847a7..3f8949d0b7 100644 --- a/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py +++ b/src/sqlbuild/compiler/discovery/classes/directory_snapshot.py @@ -2,9 +2,7 @@ from __future__ import annotations -import fnmatch import os -import re from collections.abc import Callable, Iterator from contextlib import contextmanager from contextvars import ContextVar, Token @@ -12,6 +10,7 @@ from pathlib import Path from typing import ClassVar +import sqlbuild._native as _native from sqlbuild.compiler.discovery.types import DirectorySnapshotEntry _CASE_PROBE: str = "A" @@ -138,6 +137,5 @@ def _is_dir(*, entry: os.DirEntry[str], follow_symlinks: bool) -> bool: def _name_matcher(pattern: str) -> Callable[[str], bool]: """Match names like ``Path.glob`` with the platform's default case sensitivity.""" - flags: int = 0 if os.path.normcase(_CASE_PROBE) == _CASE_PROBE else re.IGNORECASE - compiled: re.Pattern[str] = re.compile(fnmatch.translate(pattern), flags) - return lambda name: compiled.match(name) is not None + ignore_case: bool = os.path.normcase(_CASE_PROBE) != _CASE_PROBE + return lambda name: _native.glob_name_match(name, pattern, ignore_case) diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index f41b35b72b..0d61af07f3 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -186,14 +186,6 @@ ) SQL_HOOK_IDENTITY_FIELDS: tuple[str, ...] = ("statement", "name", "definition_sql", "kwargs") -STATEMENT_HEADER_BODY_PATTERN: str = ( - r"(?P
(?>(?:[^\"')\\]|\\.)++" - r"|\"(?:[^\"\\]|\\.)*+\"" - r"|'(?:[^'\\]|\\.)*+'" - r"|[\"'](?:[^\\)]|\\.)*+" - r"|\)(?!\s*;))*+)" -) - NATIVE_FAILED_TAG: str = "error" NATIVE_UNREADABLE_TAG: str = "read" NATIVE_UNLISTABLE_TAG: str = "unlistable" diff --git a/src/sqlbuild/compiler/discovery/main/_model_header_entry_span.py b/src/sqlbuild/compiler/discovery/main/_model_header_entry_span.py deleted file mode 100644 index 70860d1a13..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_model_header_entry_span.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public authored MODEL header entry span query.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - model_header_entry_span as _model_header_entry_span, -) - - -def get_model_header_entry_span(*, contents: str, key: str) -> tuple[int, int] | None: - """Return authored byte offsets for one top-level `key value` MODEL header entry.""" - - return _model_header_entry_span(contents=contents, key=key) diff --git a/src/sqlbuild/compiler/discovery/main/_model_header_spans.py b/src/sqlbuild/compiler/discovery/main/_model_header_spans.py deleted file mode 100644 index 62cb30eccd..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_model_header_spans.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public authored MODEL header span queries.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - model_header_body_span as _model_header_body_span, -) -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - model_header_columns_span as _model_header_columns_span, -) -from sqlbuild.compiler.discovery.models import ModelHeaderSpans - - -def get_model_header_spans(*, contents: str) -> ModelHeaderSpans: - """Return authored byte offsets for the MODEL header and column declarations.""" - - return ModelHeaderSpans( - body=_model_header_body_span(contents=contents), - columns=_model_header_columns_span(contents=contents), - ) diff --git a/src/sqlbuild/compiler/discovery/main/_model_query_start.py b/src/sqlbuild/compiler/discovery/main/_model_query_start.py deleted file mode 100644 index c0411a93f8..0000000000 --- a/src/sqlbuild/compiler/discovery/main/_model_query_start.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Locate a model's discovered query SQL within its authored file.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery._helpers.sql.model_files import match_model_header -from sqlbuild.compiler.discovery.models import ModelHeaderMatch - - -def get_model_query_start(*, contents: str, query_sql: str) -> int | None: - """Return where discovery's stripped query SQL starts after the MODEL header, if it does.""" - - header_match: ModelHeaderMatch | None = match_model_header(contents) - if header_match is None: - return None - sql: str = header_match.sql - start: int = header_match.sql_start + len(sql) - len(sql.lstrip()) - return start if contents.startswith(query_sql, start) else None diff --git a/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py b/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py index f3fe5ab0d0..30fa09e4a7 100644 --- a/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py +++ b/src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py @@ -2,10 +2,18 @@ from __future__ import annotations -from sqlbuild.compiler.discovery._helpers.yml.entry_lines import yaml_entry_lines +from collections.abc import Mapping +from functools import lru_cache + +import sqlbuild._native as _native def yaml_entry_line(*, contents: str, name: str) -> int | None: """Return the 1-based line of the outermost `- name: ` entry, if present.""" - return yaml_entry_lines(contents).get(name) + return _entry_lines(contents).get(name) + + +@lru_cache(maxsize=16) +def _entry_lines(contents: str) -> Mapping[str, int]: + return dict(_native.yaml_entry_line_index(contents)) diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 82cf620cea..fdf22d8355 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -85,46 +85,6 @@ class DiscoveredSqlModelFile: extract_implicit_alias_columns: bool = True -@dataclass(frozen=True) -class ModelHeaderMatch: - """Offsets of one matched ``MODEL(...)`` header within its file contents.""" - - contents: str - header_start: int - header_end: int - sql_start: int - - @property - def header(self) -> str: - """Return the header body between the MODEL parentheses.""" - - return self.contents[self.header_start : self.header_end] - - @property - def sql(self) -> str: - """Return everything after the header terminator and following whitespace.""" - - return self.contents[self.sql_start :] - - -@dataclass(frozen=True) -class ModelHeaderColumnSpan: - """Authored byte offsets for one MODEL(columns) entry and metadata body.""" - - entry_start: int - entry_end: int - metadata_start: int - metadata_end: int - - -@dataclass(frozen=True) -class ModelHeaderSpans: - """Authored MODEL header body and optional columns declaration spans.""" - - body: tuple[int, int] | None - columns: tuple[int, int, dict[str, ModelHeaderColumnSpan]] | None - - @dataclass(frozen=True) class DiscoveredSqlFunctionFile: """A discovered SQL function file and its raw contents.""" diff --git a/src/sqlbuild/compiler/fingerprints/_helpers/query.py b/src/sqlbuild/compiler/fingerprints/_helpers/query.py index c45a691ca7..8ec44bddb6 100644 --- a/src/sqlbuild/compiler/fingerprints/_helpers/query.py +++ b/src/sqlbuild/compiler/fingerprints/_helpers/query.py @@ -1,37 +1,24 @@ -"""Query fingerprint normalization and hashing.""" +"""Query fingerprint normalization and hashing, computed natively.""" from __future__ import annotations -import hashlib -import re from functools import lru_cache import sqlbuild._native as _native -from sqlbuild.compiler.fingerprints.constants import ( - GENERIC_FINGERPRINT_DIALECT, - QUERY_FINGERPRINT_CACHE_SIZE, - SQL_FUNCTION_LANGUAGE, -) +from sqlbuild.compiler.fingerprints.constants import QUERY_FINGERPRINT_CACHE_SIZE from sqlbuild.compiler.fingerprints.exceptions import QueryFingerprintError -from sqlbuild.compiler.sql_analysis.constants import NATIVE_DIALECT_ALIASES - -_WHITESPACE_RUN: re.Pattern[str] = re.compile(r"\s+") def normalize_query_sql_impl(query_sql: str) -> str: """Normalize query SQL text by collapsing whitespace runs.""" - stripped: str = query_sql.strip() - return _WHITESPACE_RUN.sub(" ", stripped) + return _native.normalize_query_text(query_sql) def compute_query_hash_impl(*, query_sql: str, dialect: str | None) -> str: """Compute the persisted SHA-256 fingerprint of the query's canonical token stream.""" - native_dialect: str = ( - NATIVE_DIALECT_ALIASES.get(dialect, dialect) if dialect else GENERIC_FINGERPRINT_DIALECT - ) - return _cached_query_hash(query_sql=query_sql, dialect=native_dialect) + return _cached_query_hash(query_sql=query_sql, dialect=dialect) def compute_function_definition_hash_impl( @@ -39,14 +26,15 @@ def compute_function_definition_hash_impl( ) -> str: """Fingerprint SQL function definitions like queries and other languages exactly.""" - if language == SQL_FUNCTION_LANGUAGE: - return compute_query_hash_impl(query_sql=fingerprint_sql, dialect=dialect) - return hashlib.sha256(fingerprint_sql.encode("utf-8")).hexdigest() + try: + return _native.function_definition_hash(fingerprint_sql, language, dialect) + except ValueError as error: + raise QueryFingerprintError(str(error)) from None @lru_cache(maxsize=QUERY_FINGERPRINT_CACHE_SIZE) -def _cached_query_hash(*, query_sql: str, dialect: str) -> str: +def _cached_query_hash(*, query_sql: str, dialect: str | None) -> str: try: - return _native.query_fingerprint(query_sql, dialect) + return _native.compiler_query_hash(query_sql, dialect) except ValueError as error: raise QueryFingerprintError(str(error)) from None diff --git a/src/sqlbuild/compiler/fingerprints/constants.py b/src/sqlbuild/compiler/fingerprints/constants.py index b2e7b2d999..2c8986375b 100644 --- a/src/sqlbuild/compiler/fingerprints/constants.py +++ b/src/sqlbuild/compiler/fingerprints/constants.py @@ -31,6 +31,4 @@ NODE_TYPE_QUERY_DIFF_ARTIFACT: str = "query_diff_artifact" AUDIT_GATE_METADATA_KEY: str = "audit_gate" -GENERIC_FINGERPRINT_DIALECT: str = "generic" QUERY_FINGERPRINT_CACHE_SIZE: int = 16_384 -SQL_FUNCTION_LANGUAGE: str = "sql" diff --git a/src/sqlbuild/compiler/frontier/_helpers/code_identity.py b/src/sqlbuild/compiler/frontier/_helpers/code_identity.py index b173a9509d..6886e97c61 100644 --- a/src/sqlbuild/compiler/frontier/_helpers/code_identity.py +++ b/src/sqlbuild/compiler/frontier/_helpers/code_identity.py @@ -2,21 +2,18 @@ from __future__ import annotations -import hashlib import json -import os import platform from functools import cache from importlib.metadata import PackageNotFoundError, distribution from pathlib import Path -from typing import Any + +import sqlbuild._native as _native _PACKAGE_NAME: str = "sqlbuild" _POLYGLOT_PACKAGE_NAME: str = "polyglot-sql-chio" _YAML_PACKAGE_NAME: str = "PyYAML" _DIRECT_URL_FILE: str = "direct_url.json" -_SOURCE_SUFFIXES: frozenset[str] = frozenset({".py", ".so", ".pyd"}) -_BYTECODE_CACHE_DIRECTORY: str = "__pycache__" _PACKAGE_ROOT_PARENT_DEPTH: int = 3 @@ -24,44 +21,21 @@ def installed_code_identity() -> str: """Return a digest that changes with every released or locally edited code version.""" - digest: Any = hashlib.sha256() - for item in ( + facts: list[str] = [ _package_version(_PACKAGE_NAME), _package_version(_POLYGLOT_PACKAGE_NAME), _package_version(_YAML_PACKAGE_NAME), platform.python_implementation(), ".".join(platform.python_version_tuple()[:2]), - ): - digest.update(item.encode()) - digest.update(b"\0") - if _is_editable_install(): - package_root: Path = Path(__file__).resolve().parents[_PACKAGE_ROOT_PARENT_DEPTH] - for relative_path, size, mtime_ns in _source_file_stats(root=package_root): - digest.update(f"{relative_path}\0{size}\0{mtime_ns}\0".encode()) - return str(digest.hexdigest()) - - -def _source_file_stats(*, root: Path) -> list[tuple[str, int, int]]: - root_prefix_length: int = len(str(root)) + 1 - stats: list[tuple[str, int, int]] = [] - pending: list[str] = [str(root)] - while pending: - directory: str = pending.pop() - try: - with os.scandir(directory) as entries: - for entry in entries: - if entry.is_dir(follow_symlinks=False): - if entry.name != _BYTECODE_CACHE_DIRECTORY: - pending.append(entry.path) - elif os.path.splitext(entry.name)[1] in _SOURCE_SUFFIXES: - stat: os.stat_result = entry.stat() - stats.append( - (entry.path[root_prefix_length:], stat.st_size, stat.st_mtime_ns) - ) - except OSError: - continue - stats.sort() - return stats + ] + package_root: Path | None = ( + Path(__file__).resolve().parents[_PACKAGE_ROOT_PARENT_DEPTH] + if _is_editable_install() + else None + ) + return _native.installed_code_identity( + facts, None if package_root is None else str(package_root) + ) def _package_version(package: str) -> str: diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py index d7b40356cc..ad5a2de920 100644 --- a/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py +++ b/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py @@ -18,11 +18,15 @@ from typing import Any from sqlbuild.compiler.frontier.constants import ( + STAGE_CAPTURE_CANONICAL_TYPES, STAGE_CAPTURE_DECODED_SEQUENCES, STAGE_CAPTURE_ENGINE_NAMESPACE_PATTERN, STAGE_CAPTURE_INVOCATION_ID_MASK, STAGE_CAPTURE_INVOCATION_ID_PATTERN, STAGE_CAPTURE_OMITTED_ATTRIBUTES, + STAGE_CAPTURE_PROJECTED_ATTRIBUTES, + STAGE_CAPTURE_REFACTOR_STAGING_MASK, + STAGE_CAPTURE_REFACTOR_STAGING_PATTERN, STAGE_CAPTURE_RESERVED_KEYS, STAGE_CAPTURE_SHARED_MARKER, STAGE_CAPTURE_SHARED_MIN_BYTES, @@ -126,13 +130,18 @@ def _encode_container(self, value: object) -> object: "args": self.encode(value.args), "keywords": self.encode(value.keywords), } - attributes: dict[str, object] | None = _instance_attributes(value) + projected: tuple[str, ...] | None = STAGE_CAPTURE_PROJECTED_ATTRIBUTES.get(type_name) + attributes: dict[str, object] | None = ( + _instance_attributes(value) + if projected is None + else {name: getattr(value, name) for name in projected} + ) if attributes is None: return {"__opaque__": type_name} unordered: frozenset[str] = STAGE_CAPTURE_UNORDERED_ATTRIBUTES.get(type_name, frozenset()) omitted: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get(type_name, frozenset()) return { - "__type__": type_name, + "__type__": STAGE_CAPTURE_CANONICAL_TYPES.get(type_name, type_name), **{ name: self._encode_attribute(value=item, unordered=name in unordered) for name, item in attributes.items() @@ -186,12 +195,13 @@ def _encode_scalar(value: object) -> object | None: def mask_capture_noise(text: str) -> str: - """Mask invocation ids and native store suffixes, which differ between identical compiles.""" + """Mask invocation ids, native store suffixes and refactor staging folders of a compile.""" - return STAGE_CAPTURE_ENGINE_NAMESPACE_PATTERN.sub( + masked: str = STAGE_CAPTURE_ENGINE_NAMESPACE_PATTERN.sub( "", STAGE_CAPTURE_INVOCATION_ID_PATTERN.sub(STAGE_CAPTURE_INVOCATION_ID_MASK, text), ) + return STAGE_CAPTURE_REFACTOR_STAGING_PATTERN.sub(STAGE_CAPTURE_REFACTOR_STAGING_MASK, masked) def _sort_key(item: object) -> str: diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 6dad7ec4e2..307417a8d6 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -16,7 +16,28 @@ CompilerEngine.NATIVE_PREVIEW: NATIVE_PREVIEW_CACHE_NAMESPACE_SUFFIX, } RETIRED_CACHE_NAMESPACE_SUFFIXES: tuple[str, ...] = ("", *ENGINE_CACHE_NAMESPACE_SUFFIXES.values()) -NATIVE_STAGE_TIERS: dict[NativeStage, NativeStageTier] = {} +NATIVE_STAGE_TIERS: dict[NativeStage, NativeStageTier] = { + NativeStage.DISCOVERY: NativeStageTier.SHIPPED, + NativeStage.DECLARATION_SCOPES: NativeStageTier.SHIPPED, + NativeStage.MODEL_CONFIG: NativeStageTier.SHIPPED, + NativeStage.REFERENCE_EXTRACTION: NativeStageTier.SHIPPED, + NativeStage.DECLARATION_FILES: NativeStageTier.SHIPPED, + NativeStage.MODEL_LOOP: NativeStageTier.SHIPPED, + NativeStage.MACRO_CALLS: NativeStageTier.SHIPPED, + NativeStage.MACRO_CALL_STORE: NativeStageTier.SHIPPED, + NativeStage.ATTACHMENTS: NativeStageTier.SHIPPED, + NativeStage.TYPE_SYSTEM: NativeStageTier.SHIPPED, + NativeStage.MODEL_ANALYSIS: NativeStageTier.SHIPPED, + NativeStage.SEMANTIC_CHECKS: NativeStageTier.SHIPPED, + NativeStage.CONTRACTS: NativeStageTier.SHIPPED, + NativeStage.LINEAGE_FACTS: NativeStageTier.SHIPPED, + NativeStage.SQL_TEST_GLUE: NativeStageTier.SHIPPED, + NativeStage.PROJECT_ASSEMBLY: NativeStageTier.SHIPPED, + NativeStage.COMPILE_LINT_INPUTS: NativeStageTier.SHIPPED, + NativeStage.RULES_REQUEST: NativeStageTier.SHIPPED, + NativeStage.COMPILE_OUTPUTS: NativeStageTier.SHIPPED, + NativeStage.REFACTORING: NativeStageTier.SHIPPED, +} ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { CompilerEngine.NATIVE: frozenset({NativeStageTier.SHIPPED}), CompilerEngine.NATIVE_PREVIEW: frozenset({NativeStageTier.SHIPPED, NativeStageTier.PREVIEW}), @@ -40,20 +61,261 @@ STAGE_CAPTURE_INVOCATION_ID_PATTERN: re.Pattern[str] = re.compile(r"\b\d{8}T\d{6}Z_[0-9a-f]{12}\b") STAGE_CAPTURE_INVOCATION_ID_MASK: str = "" STAGE_CAPTURE_ENGINE_NAMESPACE_PATTERN: re.Pattern[str] = re.compile(r"-native(?:-preview)?-v\d+\b") +STAGE_CAPTURE_REFACTOR_STAGING_PATTERN: re.Pattern[str] = re.compile(r"\bsqb-refactor-[0-9a-z_]+") +STAGE_CAPTURE_REFACTOR_STAGING_MASK: str = "sqb-refactor-" STAGE_CAPTURE_SKIPPED_SLOTS: frozenset[str] = frozenset({"__dict__", "__weakref__"}) STAGE_CAPTURE_UNORDERED_ATTRIBUTES: dict[str, frozenset[str]] = { "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog": frozenset( {"schemas", "analysis_shapes", "expression_shapes"} ) } +STAGE_CAPTURE_PROJECTED_ATTRIBUTES: dict[str, tuple[str, ...]] = { + "sqlbuild._native:NativeAuditInput": ( + "audit_file", + "audit_block", + "sql_body", + "evaluation_mode", + "measurement_contract", + "thresholds", + "minimum_samples", + "measure_sql", + "evidence_sql", + "evidence_limit", + "references", + "attached_target_kind", + "attached_target_name", + "attached_column_name", + "severity", + "run_scope", + "always_run", + "description", + "declaration_usages", + "name", + ), + "sqlbuild._native:NativeCompiledAudit": ( + "key", + "scope_deps", + "name", + "definition_name", + "audit_file", + "audit_block", + "sql_body", + "evaluation_mode", + "measurement_contract", + "thresholds", + "minimum_samples", + "measure_sql", + "evidence_sql", + "evidence_limit", + "references", + "attached_target_kind", + "attached_target_name", + "attached_column_name", + "severity", + "run_scope", + "always_run", + "description", + ), + "sqlbuild._native:NativeFunctionInput": ( + "function_file", + "name", + "arguments", + "returns", + "body_sql", + "return_columns", + "references", + "database", + "schema", + "logical_database", + "logical_schema", + "fingerprint_database", + "fingerprint_schema", + "fingerprint_logical_database", + "fingerprint_logical_schema", + "language", + "runtime_version", + "entry_point", + "packages", + "declaration_usages", + "tags", + "description", + ), + "sqlbuild._native:NativeCompiledFunction": ( + "key", + "deps", + "name", + "relative_path", + "arguments", + "returns", + "body_sql", + "destination", + "fingerprint_destination", + "return_columns", + "references", + "language", + "source_file_path", + "runtime_version", + "entry_point", + "packages", + "tags", + "description", + ), + "sqlbuild._native:NativeSourceInput": ( + "source_entry", + "source_file", + "declaration_usages", + ), + "sqlbuild._native:NativeCompiledSource": ( + "key", + "deps", + "name", + "source_entry", + "source_file", + ), + "sqlbuild._native:NativeSeedInput": ("seed_file", "schema_entry", "schema_file"), + "sqlbuild._native:NativeCompiledSeed": ( + "key", + "deps", + "name", + "seed_file", + "schema_entry", + "schema_file", + "destination", + "external", + ), + "sqlbuild._native:NativeInputsView": ( + "project_config", + "local_config", + "discovered_inputs", + "sql_lexical_syntax", + "run_id", + "effective_target_name", + "effective_target", + "compile_cache_dir", + "effective_connection", + "effective_settings", + "no_sql_validation", + "effective_vars", + "macro_context", + "loaded_macros", + "public_enums", + "public_constants", + "model_inputs", + "seed_inputs", + "source_inputs", + "sql_function_inputs", + "test_inputs", + "scenario_inputs", + "audit_inputs", + "diagnostics", + "external_sql_reference_resolver", + "scope_index", + "declaration_scope", + ), + "sqlbuild._native:NativeProjectView": ( + "run_id", + "effective_target_name", + "effective_connection", + "effective_vars", + "binding_catalog", + "effective_target_database", + "effective_target_schema", + "sql_analysis_dialect", + "sql_lexical_syntax", + "compile_cache_dir", + "settings", + "scenario", + "models", + "sources", + "seeds", + "functions", + "audits", + "sql_tests", + "sql_scenarios", + "loader_functions", + "hook_functions", + "enforce_explicit_references", + "sql_hook_files", + "materialization_files", + "public_enums", + "public_constants", + "loaded_macros", + "diagnostics", + "unmatched_literal_sql_relations", + "external_sql_reference_resolver", + "scope_index", + "sql_expansions", + ), + "sqlbuild._native:NativeCompiledModel": ( + "key", + "deps", + "name", + "relative_path", + "query_sql", + "config", + "destination", + "references", + "schema_entry", + "inferred_columns", + "fast_lineage_columns", + "fast_lineage_has_star", + "fast_lineage_star_resolved", + "authored_sql", + "authored_query_sql", + "output_column_locations", + "extract_implicit_alias_columns", + "macro_deps", + "enum_declarations", + "constant_declarations", + "enum_columns", + "binding_diagnostics", + "binding_validated", + "dynamic_column_contract", + "unchecked_output_columns", + "rejected_sql_analysis_opt_out", + ), + "sqlbuild._native:NativeModelInput": ( + "model_file", + "config", + "query_sql", + "macro_source_sql", + "references", + "schema_entry", + "schema_file", + "sql_validation_enabled", + "rejected_sql_analysis_opt_out", + "enum_declarations", + "constant_declarations", + "enum_columns", + "macro_deps", + "macro_usages", + "declaration_usages", + "sql_expansion", + "lint_expansion", + ), +} STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] = { + "sqlbuild.compiler.scopes.models:ScopeLookup": frozenset({"native"}), "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog": frozenset( {"shared_analyses"} ), "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs": frozenset({"native_session"}), "sqlbuild.compiler.compile.models:DeclarationScopeResolver": frozenset({"native_contexts"}), "sqlbuild.compiler.compile.models:DeclarationScopeBuild": frozenset({"sql_test_scans"}), - "sqlbuild.compiler.compile.models:CompiledProject": frozenset({"native_session"}), + "sqlbuild.compiler.compile.models:CompileProjectInputs": frozenset({"native_project"}), + "sqlbuild.compiler.compile.models:CompiledProject": frozenset( + {"native_session", "native_project", "lineage_graph"} + ), +} +STAGE_CAPTURE_CANONICAL_TYPES: dict[str, str] = { + "sqlbuild._native:NativeInputsView": "sqlbuild.compiler.compile.models:CompileProjectInputs", + "sqlbuild._native:NativeProjectView": "sqlbuild.compiler.compile.models:CompiledProject", + "sqlbuild._native:NativeModelInput": "sqlbuild.compiler.compile.models:CompileModelInput", + "sqlbuild._native:NativeCompiledModel": "sqlbuild.compiler.compile.models:CompiledModel", + "sqlbuild._native:NativeCompiledSource": "sqlbuild.compiler.compile.models:CompiledSource", + "sqlbuild._native:NativeCompiledSeed": "sqlbuild.compiler.compile.models:CompiledSeed", + "sqlbuild._native:NativeCompiledFunction": "sqlbuild.compiler.compile.models:CompiledFunction", + "sqlbuild._native:NativeCompiledAudit": "sqlbuild.compiler.compile.models:CompiledAudit", } STAGE_CAPTURE_DECODED_SEQUENCES: frozenset[str] = frozenset( {"sqlbuild.compiler.compile.models:CompactLineageFacts"} diff --git a/src/sqlbuild/compiler/frontier/main/compiled_code_identity.py b/src/sqlbuild/compiler/frontier/main/compiled_code_identity.py index 103cd40e4f..4f4bcb6999 100644 --- a/src/sqlbuild/compiler/frontier/main/compiled_code_identity.py +++ b/src/sqlbuild/compiler/frontier/main/compiled_code_identity.py @@ -1,7 +1,5 @@ """Identity of the installed and native code behind cached compiler and rule results.""" -import hashlib - import sqlbuild._native as _native from sqlbuild.compiler.frontier._helpers.code_identity import installed_code_identity @@ -9,6 +7,4 @@ def compiled_code_identity() -> str: """Return the digest that changes with every released, locally edited, or rebuilt version.""" - return hashlib.sha256( - f"{installed_code_identity()}\0{_native.BUILD_IDENTITY}".encode() - ).hexdigest() + return _native.compiled_code_identity(installed_code_identity(), _native.BUILD_IDENTITY) diff --git a/src/sqlbuild/compiler/frontier/main/_report_native_answer.py b/src/sqlbuild/compiler/frontier/main/report_native_answer.py similarity index 100% rename from src/sqlbuild/compiler/frontier/main/_report_native_answer.py rename to src/sqlbuild/compiler/frontier/main/report_native_answer.py diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 892b1502c6..b4aea581db 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -11,7 +11,28 @@ class CompilerEngine(StrEnum): class NativeStage(StrEnum): - """A native implementation behind the preview engine until its tier is shipped.""" + """A native implementation that replaces one Python compiler stage when its tier is active.""" + + DISCOVERY = "discovery" + DECLARATION_SCOPES = "declaration_scopes" + MODEL_CONFIG = "model_config" + REFERENCE_EXTRACTION = "reference_extraction" + DECLARATION_FILES = "declaration_files" + MODEL_LOOP = "model_loop" + MACRO_CALLS = "macro_calls" + MACRO_CALL_STORE = "macro_call_store" + ATTACHMENTS = "attachments" + TYPE_SYSTEM = "type_system" + MODEL_ANALYSIS = "model_analysis" + SEMANTIC_CHECKS = "semantic_checks" + CONTRACTS = "contracts" + LINEAGE_FACTS = "lineage_facts" + SQL_TEST_GLUE = "sql_test_glue" + PROJECT_ASSEMBLY = "project_assembly" + COMPILE_LINT_INPUTS = "compile_lint_inputs" + RULES_REQUEST = "rules_request" + COMPILE_OUTPUTS = "compile_outputs" + REFACTORING = "refactoring" class NativeStageTier(StrEnum): diff --git a/src/sqlbuild/compiler/graph/_helpers/audit_gates.py b/src/sqlbuild/compiler/graph/_helpers/audit_gates.py index 42df368146..193ed835e1 100644 --- a/src/sqlbuild/compiler/graph/_helpers/audit_gates.py +++ b/src/sqlbuild/compiler/graph/_helpers/audit_gates.py @@ -4,8 +4,13 @@ from dataclasses import dataclass -from sqlbuild.compiler.compile.models import CompiledAudit, CompiledObjectKey, CompiledProject -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import ( + AttachedAuditTargetKind, + CompiledAuditView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.graph.models import AttachedAuditGateEdge _READABLE_RESOURCE_TYPES: frozenset[CompiledResourceType] = frozenset( @@ -28,7 +33,7 @@ class _AuditGate: def attached_audit_gate_edges_impl( - *, project: CompiledProject + *, project: CompiledProjectView ) -> tuple[AttachedAuditGateEdge, ...]: """Return edges from each extra audit read to its model or seed target, or source triggers.""" @@ -58,9 +63,9 @@ def attached_audit_gate_edges_impl( return tuple(edges) -def _audit_gates(*, project: CompiledProject) -> tuple[_AuditGate, ...]: +def _audit_gates(*, project: CompiledProjectView) -> tuple[_AuditGate, ...]: gates: list[_AuditGate] = [] - audit: CompiledAudit + audit: CompiledAuditView for audit in project.audits: if audit.attached_target_kind is None or audit.attached_target_name is None: continue @@ -101,7 +106,7 @@ def _gate_edges( def _extra_reads( - *, audit: CompiledAudit, target: CompiledObjectKey + *, audit: CompiledAuditView, target: CompiledObjectKey ) -> tuple[CompiledObjectKey, ...]: return tuple( read diff --git a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py index ec222b1b90..003c47ceb4 100644 --- a/src/sqlbuild/compiler/graph/_helpers/hook_reads.py +++ b/src/sqlbuild/compiler/graph/_helpers/hook_reads.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry from sqlbuild.compiler.graph._helpers.sql_refs import sql_ref_key_impl from sqlbuild.compiler.graph.models import HookReadEdge @@ -14,7 +14,7 @@ _MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") -def model_python_hook_names_impl(model: CompiledModel) -> tuple[str, ...]: +def model_python_hook_names_impl(model: CompiledModelView) -> tuple[str, ...]: """Return the Python hooks attached to one model, in lifecycle order.""" names: list[str] = [] @@ -27,7 +27,7 @@ def model_python_hook_names_impl(model: CompiledModel) -> tuple[str, ...]: return tuple(dict.fromkeys(names)) -def hook_read_edges_impl(*, project: CompiledProject) -> tuple[HookReadEdge, ...]: +def hook_read_edges_impl(*, project: CompiledProjectView) -> tuple[HookReadEdge, ...]: """Return edges from each hook read to the model that runs the hook.""" reads_by_hook: dict[str, tuple[SqlResourceRef, ...]] = { @@ -52,7 +52,7 @@ def hook_read_edges_impl(*, project: CompiledProject) -> tuple[HookReadEdge, ... return tuple(dict.fromkeys(edge for edge in candidates if edge.read != edge.gated)) -def _sql_hook_read_edges(model: CompiledModel) -> tuple[HookReadEdge, ...]: +def _sql_hook_read_edges(model: CompiledModelView) -> tuple[HookReadEdge, ...]: edges: list[HookReadEdge] = [] for hook_key in _MODEL_HOOK_KEYS: entries: object = model.config.values.get(hook_key) diff --git a/src/sqlbuild/compiler/graph/_helpers/lineage.py b/src/sqlbuild/compiler/graph/_helpers/lineage.py deleted file mode 100644 index 4487793860..0000000000 --- a/src/sqlbuild/compiler/graph/_helpers/lineage.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Compiled project lineage edge implementations.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledSeed, - CompiledSource, -) -from sqlbuild.compiler.compile.types import CompiledResourceType - - -def build_lineage_upstream_deps_impl( - project: CompiledProject, -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Build pure data-flow upstream edges with SQL tests stripped.""" - - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = {} - model: CompiledModel - for model in project.models: - upstream[model.key] = model.deps - source: CompiledSource - for source in project.sources: - upstream[source.key] = source.deps - seed: CompiledSeed - for seed in project.seeds: - upstream[seed.key] = seed.deps - function: CompiledFunction - for function in project.functions: - upstream[function.key] = function.deps - return _filter_lineage_deps(upstream) - - -def build_lineage_downstream_deps_impl( - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Build downstream edges keyed by upstream object key.""" - - downstream: dict[CompiledObjectKey, list[CompiledObjectKey]] = {} - for key in upstream: - downstream.setdefault(key, []) - for key, dep_keys in upstream.items(): - for dep_key in dep_keys: - downstream.setdefault(dep_key, []).append(key) - return { - key: tuple(sorted(values, key=lambda obj: (str(obj.resource_type), obj.name))) - for key, values in downstream.items() - } - - -def _filter_lineage_deps( - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - filtered: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = {} - for key, deps in upstream_deps.items(): - if key.resource_type == CompiledResourceType.SQL_TEST: - continue - lineage_deps: list[CompiledObjectKey] = [] - for dep in deps: - if dep.resource_type != CompiledResourceType.SQL_TEST: - lineage_deps.append(dep) - filtered[key] = tuple(lineage_deps) - return filtered diff --git a/src/sqlbuild/compiler/graph/_helpers/native_graph.py b/src/sqlbuild/compiler/graph/_helpers/native_graph.py new file mode 100644 index 0000000000..9821d8e8dd --- /dev/null +++ b/src/sqlbuild/compiler/graph/_helpers/native_graph.py @@ -0,0 +1,104 @@ +"""Native project graph construction and its Python dict views.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.graph.models import LineageGraphViews +from sqlbuild.compiler.graph.types import NativeKey + + +class _KeyCache: + """One `CompiledObjectKey` per native pair, so shared keys convert once.""" + + def __init__(self) -> None: + self._keys: dict[NativeKey, CompiledObjectKey] = {} + + def key(self, pair: NativeKey) -> CompiledObjectKey: + existing: CompiledObjectKey | None = self._keys.get(pair) + if existing is not None: + return existing + created: CompiledObjectKey = CompiledObjectKey( + resource_type=CompiledResourceType(pair[0]), name=pair[1] + ) + self._keys[pair] = created + return created + + def keys(self, pairs: Iterable[NativeKey]) -> tuple[CompiledObjectKey, ...]: + return tuple(self.key(pair) for pair in pairs) + + +def native_key(key: CompiledObjectKey) -> NativeKey: + """The `(resource type, name)` pair the native graph keys resources by.""" + + return (str(key.resource_type), key.name) + + +def native_keys(keys: Iterable[CompiledObjectKey]) -> list[NativeKey]: + """Native pairs for `keys`, in order.""" + + return [native_key(key) for key in keys] + + +def build_native_project_graph_impl(project: CompiledProjectView) -> _native.NativeProjectGraph: + """The project's native graph, built once per compiled project.""" + + return project.lineage_graph + + +def project_lineage_views_impl(project: CompiledProjectView) -> LineageGraphViews: + """The project's lineage edges and selector indexes as fresh dicts.""" + + return lineage_graph_views_impl(project.lineage_graph) + + +def native_graph_from_views_impl( + *, + all_keys: Mapping[str, CompiledObjectKey], + upstream: Mapping[CompiledObjectKey, Iterable[CompiledObjectKey]], + downstream: Mapping[CompiledObjectKey, Iterable[CompiledObjectKey]], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], +) -> _native.NativeProjectGraph: + """A native graph over dict indexes a caller built, keeping their order.""" + + return _native.NativeProjectGraph.from_indexes( + ( + [(name, native_key(key)) for name, key in all_keys.items()], + [(native_key(key), native_keys(deps)) for key, deps in upstream.items()], + [(native_key(key), native_keys(deps)) for key, deps in downstream.items()], + [(tag, native_keys(keys)) for tag, keys in tag_index.items()], + [(native_key(key), folder) for key, folder in path_index.items()], + ) + ) + + +def lineage_graph_views_impl(graph: _native.NativeProjectGraph) -> LineageGraphViews: + """The native graph's indexes as the dicts Python graph consumers read.""" + + keys: _KeyCache = _KeyCache() + names, upstream, downstream, tags, paths = graph.indexes() + return LineageGraphViews( + upstream_deps={keys.key(key): keys.keys(deps) for key, deps in upstream}, + downstream_deps={keys.key(key): keys.keys(deps) for key, deps in downstream}, + tag_index={tag: frozenset(keys.keys(tagged)) for tag, tagged in tags}, + path_index={keys.key(key): folder for key, folder in paths}, + all_keys={name: keys.key(key) for name, key in names}, + ) + + +def lineage_graph_names_impl(graph: _native.NativeProjectGraph) -> dict[str, CompiledObjectKey]: + """Selector name to key, converted from the native names only.""" + + keys: _KeyCache = _KeyCache() + return {name: keys.key(key) for name, key in graph.names()} + + +def python_keys(keys: Iterable[NativeKey]) -> frozenset[CompiledObjectKey]: + """`CompiledObjectKey`s for native pairs.""" + + cache: _KeyCache = _KeyCache() + return frozenset(cache.keys(keys)) diff --git a/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py b/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py index e62bff0021..d9ab03745d 100644 --- a/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py +++ b/src/sqlbuild/compiler/graph/main/_attached_audit_gate_edges.py @@ -2,12 +2,12 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.graph._helpers.audit_gates import attached_audit_gate_edges_impl from sqlbuild.compiler.graph.models import AttachedAuditGateEdge -def attached_audit_gate_edges(*, project: CompiledProject) -> tuple[AttachedAuditGateEdge, ...]: +def attached_audit_gate_edges(*, project: CompiledProjectView) -> tuple[AttachedAuditGateEdge, ...]: """Return edges from resources an attached audit reads to the nodes the audit gates.""" return attached_audit_gate_edges_impl(project=project) diff --git a/src/sqlbuild/compiler/graph/main/_build_lineage_downstream_deps.py b/src/sqlbuild/compiler/graph/main/_build_lineage_downstream_deps.py deleted file mode 100644 index b8b3ef254c..0000000000 --- a/src/sqlbuild/compiler/graph/main/_build_lineage_downstream_deps.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Compiled project downstream lineage entrypoint.""" - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.graph._helpers.lineage import build_lineage_downstream_deps_impl - - -def build_lineage_downstream_deps( - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Return downstream edges keyed by upstream object key.""" - - return build_lineage_downstream_deps_impl(upstream) diff --git a/src/sqlbuild/compiler/graph/main/_build_lineage_upstream_deps.py b/src/sqlbuild/compiler/graph/main/_build_lineage_upstream_deps.py deleted file mode 100644 index 60204c1444..0000000000 --- a/src/sqlbuild/compiler/graph/main/_build_lineage_upstream_deps.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Compiled project upstream lineage entrypoint.""" - -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject -from sqlbuild.compiler.graph._helpers.lineage import build_lineage_upstream_deps_impl - - -def build_lineage_upstream_deps( - project: CompiledProject, -) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: - """Return pure data-flow upstream edges with SQL tests stripped.""" - - return build_lineage_upstream_deps_impl(project) diff --git a/src/sqlbuild/compiler/graph/main/_compiled_graph_keys.py b/src/sqlbuild/compiler/graph/main/_compiled_graph_keys.py new file mode 100644 index 0000000000..2fbea3c89d --- /dev/null +++ b/src/sqlbuild/compiler/graph/main/_compiled_graph_keys.py @@ -0,0 +1,13 @@ +"""Conversion of the native graph's key pairs to compiled keys.""" + +from collections.abc import Iterable + +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.graph._helpers.native_graph import python_keys +from sqlbuild.compiler.graph.types import NativeKey + + +def compiled_graph_keys(keys: Iterable[NativeKey]) -> frozenset[CompiledObjectKey]: + """Return `CompiledObjectKey`s for native graph key pairs.""" + + return python_keys(keys) diff --git a/src/sqlbuild/compiler/graph/main/_hook_read_edges.py b/src/sqlbuild/compiler/graph/main/_hook_read_edges.py index c4448480c4..3dffa67abe 100644 --- a/src/sqlbuild/compiler/graph/main/_hook_read_edges.py +++ b/src/sqlbuild/compiler/graph/main/_hook_read_edges.py @@ -2,12 +2,12 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.graph._helpers.hook_reads import hook_read_edges_impl from sqlbuild.compiler.graph.models import HookReadEdge -def hook_read_edges(*, project: CompiledProject) -> tuple[HookReadEdge, ...]: +def hook_read_edges(*, project: CompiledProjectView) -> tuple[HookReadEdge, ...]: """Return edges from resources a Python hook declares it reads to the model running it.""" return hook_read_edges_impl(project=project) diff --git a/src/sqlbuild/compiler/graph/main/_lineage_graph_names.py b/src/sqlbuild/compiler/graph/main/_lineage_graph_names.py new file mode 100644 index 0000000000..5b80438436 --- /dev/null +++ b/src/sqlbuild/compiler/graph/main/_lineage_graph_names.py @@ -0,0 +1,11 @@ +"""Selector names of a native project graph.""" + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.graph._helpers.native_graph import lineage_graph_names_impl + + +def lineage_graph_names(graph: _native.NativeProjectGraph) -> dict[str, CompiledObjectKey]: + """Return selector name to key, in project order, without the other graph views.""" + + return lineage_graph_names_impl(graph) diff --git a/src/sqlbuild/compiler/graph/main/_lineage_graph_views.py b/src/sqlbuild/compiler/graph/main/_lineage_graph_views.py new file mode 100644 index 0000000000..762324a6a4 --- /dev/null +++ b/src/sqlbuild/compiler/graph/main/_lineage_graph_views.py @@ -0,0 +1,11 @@ +"""Python dict views of a native project graph.""" + +import sqlbuild._native as _native +from sqlbuild.compiler.graph._helpers.native_graph import lineage_graph_views_impl +from sqlbuild.compiler.graph.models import LineageGraphViews + + +def lineage_graph_views(graph: _native.NativeProjectGraph) -> LineageGraphViews: + """Return the graph's lineage edges and selector indexes as dicts, in project order.""" + + return lineage_graph_views_impl(graph) diff --git a/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py b/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py index 7505329945..b80abfb8c6 100644 --- a/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py +++ b/src/sqlbuild/compiler/graph/main/_model_python_hook_names.py @@ -2,11 +2,11 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.graph._helpers.hook_reads import model_python_hook_names_impl -def model_python_hook_names(*, model: CompiledModel) -> tuple[str, ...]: +def model_python_hook_names(*, model: CompiledModelView) -> tuple[str, ...]: """Return the Python hooks attached to one model, in lifecycle order.""" return model_python_hook_names_impl(model) diff --git a/src/sqlbuild/compiler/graph/main/_native_graph_from_edges.py b/src/sqlbuild/compiler/graph/main/_native_graph_from_edges.py new file mode 100644 index 0000000000..61127098cd --- /dev/null +++ b/src/sqlbuild/compiler/graph/main/_native_graph_from_edges.py @@ -0,0 +1,19 @@ +"""Native graph over caller-built edge maps alone.""" + +from collections.abc import Iterable, Mapping + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.graph._helpers.native_graph import native_graph_from_views_impl + + +def native_graph_from_edges( + *, + upstream: Mapping[CompiledObjectKey, Iterable[CompiledObjectKey]], + downstream: Mapping[CompiledObjectKey, Iterable[CompiledObjectKey]], +) -> _native.NativeProjectGraph: + """Return a native graph holding these edges in their order, with no selector indexes.""" + + return native_graph_from_views_impl( + all_keys={}, upstream=upstream, downstream=downstream, tag_index={}, path_index={} + ) diff --git a/src/sqlbuild/compiler/graph/main/_native_graph_from_views.py b/src/sqlbuild/compiler/graph/main/_native_graph_from_views.py new file mode 100644 index 0000000000..c594e52536 --- /dev/null +++ b/src/sqlbuild/compiler/graph/main/_native_graph_from_views.py @@ -0,0 +1,26 @@ +"""Native project graph over caller-built dict indexes.""" + +from collections.abc import Iterable, Mapping + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.graph._helpers.native_graph import native_graph_from_views_impl + + +def native_graph_from_views( + *, + all_keys: Mapping[str, CompiledObjectKey], + upstream: Mapping[CompiledObjectKey, Iterable[CompiledObjectKey]], + downstream: Mapping[CompiledObjectKey, Iterable[CompiledObjectKey]], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], +) -> _native.NativeProjectGraph: + """Return a native graph over these indexes, keeping their order.""" + + return native_graph_from_views_impl( + all_keys=all_keys, + upstream=upstream, + downstream=downstream, + tag_index=tag_index, + path_index=path_index, + ) diff --git a/src/sqlbuild/compiler/graph/main/_native_graph_key.py b/src/sqlbuild/compiler/graph/main/_native_graph_key.py new file mode 100644 index 0000000000..cf06cbcd63 --- /dev/null +++ b/src/sqlbuild/compiler/graph/main/_native_graph_key.py @@ -0,0 +1,11 @@ +"""Conversion of one compiled key to the native graph's key pair.""" + +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.graph._helpers.native_graph import native_key +from sqlbuild.compiler.graph.types import NativeKey + + +def native_graph_key(key: CompiledObjectKey) -> NativeKey: + """Return the native graph's `(resource type, name)` pair for `key`.""" + + return native_key(key) diff --git a/src/sqlbuild/compiler/graph/main/_native_graph_keys.py b/src/sqlbuild/compiler/graph/main/_native_graph_keys.py new file mode 100644 index 0000000000..ff2d3baff4 --- /dev/null +++ b/src/sqlbuild/compiler/graph/main/_native_graph_keys.py @@ -0,0 +1,13 @@ +"""Conversion of compiled keys to the native graph's key pairs.""" + +from collections.abc import Iterable + +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.graph._helpers.native_graph import native_keys +from sqlbuild.compiler.graph.types import NativeKey + + +def native_graph_keys(keys: Iterable[CompiledObjectKey]) -> list[NativeKey]: + """Return the native graph's `(resource type, name)` pairs for `keys`, in order.""" + + return native_keys(keys) diff --git a/src/sqlbuild/compiler/graph/main/_native_project_graph.py b/src/sqlbuild/compiler/graph/main/_native_project_graph.py new file mode 100644 index 0000000000..5032d11637 --- /dev/null +++ b/src/sqlbuild/compiler/graph/main/_native_project_graph.py @@ -0,0 +1,11 @@ +"""Native project graph entrypoint.""" + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.graph._helpers.native_graph import build_native_project_graph_impl + + +def build_native_project_graph(project: CompiledProjectView) -> _native.NativeProjectGraph: + """Index the project's lineage edges, tags, folders and names natively.""" + + return build_native_project_graph_impl(project) diff --git a/src/sqlbuild/compiler/graph/main/project_lineage_views.py b/src/sqlbuild/compiler/graph/main/project_lineage_views.py new file mode 100644 index 0000000000..89e2ba81f5 --- /dev/null +++ b/src/sqlbuild/compiler/graph/main/project_lineage_views.py @@ -0,0 +1,11 @@ +"""Lineage dicts of a compiled project.""" + +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.graph._helpers.native_graph import project_lineage_views_impl +from sqlbuild.compiler.graph.models import LineageGraphViews + + +def project_lineage_views(project: CompiledProjectView) -> LineageGraphViews: + """Return the project's lineage edges and selector indexes, computed natively.""" + + return project_lineage_views_impl(project) diff --git a/src/sqlbuild/compiler/graph/models.py b/src/sqlbuild/compiler/graph/models.py index b6e3e3661c..9dd0c15426 100644 --- a/src/sqlbuild/compiler/graph/models.py +++ b/src/sqlbuild/compiler/graph/models.py @@ -9,6 +9,17 @@ from sqlbuild.compiler.graph.types import HookReadType +@dataclass(frozen=True) +class LineageGraphViews: + """A native project graph's lineage edges and selector indexes as Python dicts.""" + + upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] + tag_index: dict[str, frozenset[CompiledObjectKey]] + path_index: dict[CompiledObjectKey, str] + all_keys: dict[str, CompiledObjectKey] + + @dataclass(frozen=True) class AttachedAuditGateEdge: """One ordering edge: the gated node waits for a resource its attached audit reads.""" diff --git a/src/sqlbuild/compiler/graph/types.py b/src/sqlbuild/compiler/graph/types.py index 37940e899c..a1c6a43cde 100644 --- a/src/sqlbuild/compiler/graph/types.py +++ b/src/sqlbuild/compiler/graph/types.py @@ -1,4 +1,4 @@ -"""Enumerations shared by graph facts.""" +"""Enumerations and aliases shared by graph facts.""" from __future__ import annotations @@ -11,3 +11,6 @@ class HookReadType(StrEnum): PYTHON = "Python" SQL = "SQL" INLINE_SQL = "inline SQL" + + +type NativeKey = tuple[str, str] diff --git a/src/sqlbuild/compiler/lineage/_helpers/columns.py b/src/sqlbuild/compiler/lineage/_helpers/columns.py deleted file mode 100644 index ab24690c4c..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/columns.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Shared helpers for column lineage extraction.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.models import ColumnLineage, ColumnLineageSource, PhysicalResource -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, -) - -_SQLBUILD_REF_PATTERN: re.Pattern[str] = re.compile( - r"__(?Pref|source|seed)\(\s*(['\"])(?P[^'\"]+)\2\s*\)" -) -_SQLBUILD_UDF_PATTERN: re.Pattern[str] = re.compile( - r"__udf\(\s*(['\"])(?P[^'\"]+)\1\s*\)\s*\(" -) - - -def _normalize_sqlbuild_refs(sql: str) -> tuple[str, tuple[PhysicalResource, ...]]: - def replace(match: re.Match[str]) -> str: - kind: str = match.group("kind") - name: str = match.group("name") - resource_type: CompiledResourceType = { - "ref": CompiledResourceType.MODEL, - "source": CompiledResourceType.SOURCE, - "seed": CompiledResourceType.SEED, - }[kind] - physical_name: str = _physical_resource_name( - resource_type=resource_type, resource_name=name - ) - return physical_name - - resources: tuple[PhysicalResource, ...] = tuple( - PhysicalResource( - resource_type={ - "ref": CompiledResourceType.MODEL, - "source": CompiledResourceType.SOURCE, - "seed": CompiledResourceType.SEED, - }[match.group("kind")], - resource_name=match.group("name"), - physical_name=replace(match), - ) - for match in _SQLBUILD_REF_PATTERN.finditer(sql) - ) - normalized_sql: str = _SQLBUILD_REF_PATTERN.sub(replace, sql) - normalized_sql = _SQLBUILD_UDF_PATTERN.sub( - lambda match: f"{_physical_function_name(match.group('name'))}(", - normalized_sql, - ) - return normalized_sql, resources - - -def _physical_resource_name(*, resource_type: CompiledResourceType, resource_name: str) -> str: - safe_name: str = re.sub(r"[^a-zA-Z0-9_]", "__", resource_name) - return f"__sqlbuild_{resource_type.value}__{safe_name}" - - -def _physical_function_name(function_name: str) -> str: - return re.sub(r"[^a-zA-Z0-9_]", "__", function_name) - - -def _build_schema_mapping(project: CompiledProject) -> dict[str, dict[str, str]]: - schema: dict[str, dict[str, str]] = {} - for model in project.models: - columns: dict[str, str] = {} - for column in model.inferred_columns or (): - columns[column.name] = column.type or "UNKNOWN" - for column in model.schema_entry.columns if model.schema_entry is not None else (): - columns.setdefault(column.name, column.type or "UNKNOWN") - if columns: - schema[ - _physical_resource_name( - resource_type=CompiledResourceType.MODEL, resource_name=model.name - ) - ] = columns - for source in project.sources: - columns: dict[str, str] = { - column.name: column.type or "UNKNOWN" for column in source.source_entry.columns - } - if columns: - schema[ - _physical_resource_name( - resource_type=CompiledResourceType.SOURCE, resource_name=source.name - ) - ] = columns - for seed in project.seeds: - columns: dict[str, str] = { - column.name: column.type or "UNKNOWN" for column in seed.schema_entry.columns - } - if columns: - schema[ - _physical_resource_name( - resource_type=CompiledResourceType.SEED, resource_name=seed.name - ) - ] = columns - return schema - - -def _build_star_lineage( - *, - model: CompiledModel, - schema: dict[str, dict[str, str]], - physical_resources: tuple[PhysicalResource, ...], - existing_columns: set[str], -) -> tuple[ColumnLineage, ...]: - lineages: list[ColumnLineage] = [] - seen_columns: set[str] = set(existing_columns) - for resource in physical_resources: - for column_name in schema.get(resource.physical_name, {}): - if column_name in seen_columns: - continue - seen_columns.add(column_name) - lineages.append( - ColumnLineage( - output_column=column_name, - transform_kind=ColumnTransformKind.STAR, - expression_sql=None, - upstream_columns=( - ColumnLineageSource( - resource_type=resource.resource_type, - resource_name=resource.resource_name, - column_name=column_name, - ), - ), - nullability=InferredNullability.UNKNOWN, - confidence=ColumnLineageConfidence.MEDIUM, - ) - ) - return tuple(lineages) diff --git a/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py b/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py index 7f66a866c6..48e0931995 100644 --- a/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py +++ b/src/sqlbuild/compiler/lineage/_helpers/native_fast_columns.py @@ -10,10 +10,12 @@ CompactLineageFacts, CompiledLineageColumnFact, CompiledLineageSourceFact, - CompiledModel, - CompiledProject, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.lineage.constants import ( NATIVE_LINEAGE_BUILT, NATIVE_LINEAGE_UNKNOWN_DIALECT, @@ -41,7 +43,7 @@ def build_native_fast_project_column_lineage( *, - project: CompiledProject, + project: CompiledProjectView, dialect: str | None, model_names: frozenset[str] | None, ) -> ProjectColumnLineage | None: @@ -49,7 +51,7 @@ def build_native_fast_project_column_lineage( if not project.settings.sql_analysis: return None - models: tuple[CompiledModel, ...] = tuple( + models: tuple[CompiledModelView, ...] = tuple( model for model in project.models if model_names is None or model.name in model_names ) requests: list[tuple[bool, str, list[str]]] = [] @@ -108,14 +110,14 @@ def build_native_fast_project_column_lineage( ) -def _needs_star_expansion(model: CompiledModel) -> bool: +def _needs_star_expansion(model: CompiledModelView) -> bool: return model.fast_lineage_has_star and not model.fast_lineage_star_resolved -def _native_request(model: CompiledModel) -> tuple[bool, str, list[str]] | None: +def _native_request(model: CompiledModelView) -> tuple[bool, str, list[str]] | None: columns: Sequence[CompiledLineageColumnFact] | None = model.fast_lineage_columns if columns is None: - return (False, model.query_sql, [column.name for column in model.inferred_columns or ()]) + return (False, model.query_sql, [name for name, _, _ in model.inferred_rows or ()]) if not _needs_star_expansion(model): return None if isinstance(columns, CompactLineageFacts): @@ -123,12 +125,11 @@ def _native_request(model: CompiledModel) -> tuple[bool, str, list[str]] | None: return (True, model.query_sql, [column.output_column for column in columns]) -def _schema_resources(project: CompiledProject) -> list[tuple[str, str, list[str]]]: +def _schema_resources(project: CompiledProjectView) -> list[tuple[str, str, list[str]]]: resources: list[tuple[str, str, list[str]]] = [] for model in project.models: - columns: list[str] = [column.name for column in model.inferred_columns or ()] - if model.schema_entry is not None: - columns.extend(column.name for column in model.schema_entry.columns) + columns: list[str] = [name for name, _, _ in model.inferred_rows or ()] + columns.extend(name for name, _ in model.schema_column_rows or ()) resources.append((CompiledResourceType.MODEL.value, model.name, columns)) for source in project.sources: resources.append( diff --git a/src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py b/src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py new file mode 100644 index 0000000000..84f3bdd6cc --- /dev/null +++ b/src/sqlbuild/compiler/lineage/_helpers/native_rich_columns.py @@ -0,0 +1,133 @@ +"""Rich column lineage built by the native engine, assembled exactly as Python assembles it.""" + +from __future__ import annotations + +import logging + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) +from sqlbuild.compiler.lineage._helpers.rich_columns import ( + _polyglot_dialect, + assemble_rich_project_column_lineage, +) +from sqlbuild.compiler.lineage.constants import ( + NATIVE_LINEAGE_BUILT, + NATIVE_RICH_LINEAGE_SKIPPED, +) +from sqlbuild.compiler.lineage.models import ( + ColumnLineage, + ColumnLineageSource, + ModelColumnLineage, + ProjectColumnLineage, +) +from sqlbuild.compiler.lineage.types import ( + ColumnLineageConfidence, + ColumnTransformKind, + InferredNullability, +) +from sqlbuild.diagnostics.main.log_debug_event import log_debug_event + +_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.lineage") + +type _TypedColumns = list[tuple[str, str | None]] +type _NativeColumn = tuple[str, str, str, str, list[tuple[str, str, str]]] +type _NativeOutcome = tuple[str, list[_NativeColumn], bool, str | None] + + +def build_native_rich_project_column_lineage( + *, + project: CompiledProjectView, + dialect: str | None, + model_names: frozenset[str] | None, +) -> ProjectColumnLineage | None: + """Build the rich project column lineage graph with native query analysis.""" + + if not project.settings.sql_analysis: + return None + models: tuple[CompiledModelView, ...] = tuple( + model for model in project.models if model_names is None or model.name in model_names + ) + outcomes: list[_NativeOutcome] = ( + _native.build_rich_column_lineage( + ( + _polyglot_dialect(dialect).value, + _schema_resources(project), + [model.query_sql for model in models], + ) + ) + if models + else [] + ) + model_results: dict[str, ModelColumnLineage] = {} + for model, (status, columns, has_star, detail) in zip(models, outcomes, strict=True): + if status == NATIVE_LINEAGE_BUILT: + model_results[model.name] = ModelColumnLineage( + model_name=model.name, + columns=tuple(_column_lineage(column) for column in columns), + has_star=has_star, + ) + elif status == NATIVE_RICH_LINEAGE_SKIPPED: + log_debug_event( + logger=_DEBUG_LOGGER, + message="rich column lineage analysis failed; skipping model", + sqlbuild_model=model.name, + sqlbuild_error=detail, + ) + return assemble_rich_project_column_lineage(model_results) + + +def _schema_resources( + project: CompiledProjectView, +) -> list[tuple[str, str, _TypedColumns, _TypedColumns]]: + resources: list[tuple[str, str, _TypedColumns, _TypedColumns]] = [] + for model in project.models: + resources.append( + ( + CompiledResourceType.MODEL.value, + model.name, + [(name, column_type) for name, column_type, _ in model.inferred_rows or ()], + list(model.schema_column_rows or ()), + ) + ) + for source in project.sources: + resources.append( + ( + CompiledResourceType.SOURCE.value, + source.name, + [(column.name, column.type) for column in source.source_entry.columns], + [], + ) + ) + for seed in project.seeds: + resources.append( + ( + CompiledResourceType.SEED.value, + seed.name, + [(column.name, column.type) for column in seed.schema_entry.columns], + [], + ) + ) + return resources + + +def _column_lineage(column: _NativeColumn) -> ColumnLineage: + output_column, transform_kind, confidence, nullability, sources = column + return ColumnLineage( + output_column=output_column, + transform_kind=ColumnTransformKind(transform_kind), + expression_sql=None, + upstream_columns=tuple( + ColumnLineageSource( + resource_type=CompiledResourceType(resource_type), + resource_name=resource_name, + column_name=column_name, + ) + for resource_type, resource_name, column_name in sources + ), + nullability=InferredNullability(nullability), + confidence=ColumnLineageConfidence(confidence), + ) diff --git a/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py b/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py index 2cbd7b89e7..42e4c58586 100644 --- a/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py +++ b/src/sqlbuild/compiler/lineage/_helpers/rich_columns.py @@ -1,61 +1,17 @@ -"""Polyglot-backed rich column lineage analyzer.""" +"""Rich column lineage glue: the analysis dialect name and the project graph assembly.""" from __future__ import annotations -import logging -from typing import Any - from sqlbuild.adapter.contract.types import BuiltinAdapter, TypeDialect -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage._helpers.columns import ( - _build_schema_mapping, - _build_star_lineage, - _normalize_sqlbuild_refs, -) -from sqlbuild.compiler.lineage.constants import ( - POLYGLOT_RESOLVED_SOURCE_CONFIDENCE, - STAR_COLUMN_NAME, -) from sqlbuild.compiler.lineage.models import ( - ColumnLineage, ColumnLineageEdge, - ColumnLineageSource, ModelColumnLineage, - PhysicalResource, ProjectColumnLineage, QualifiedLineageColumn, ) -from sqlbuild.compiler.lineage.types import ( - ColumnLineageConfidence, - ColumnTransformKind, - InferredNullability, - PolyglotAnalysisDialect, -) -from sqlbuild.compiler.sql_analysis.constants import ( - POLYGLOT_ANALYSIS_IS_STAR, - POLYGLOT_ANALYSIS_NAME, - POLYGLOT_ANALYSIS_NULLABILITY, - POLYGLOT_ANALYSIS_NULLABILITY_NON_NULL, - POLYGLOT_ANALYSIS_NULLABILITY_NULLABLE, - POLYGLOT_ANALYSIS_PROJECTIONS, - POLYGLOT_ANALYSIS_SOURCE_ALIAS, - POLYGLOT_ANALYSIS_SOURCE_CONFIDENCE, - POLYGLOT_ANALYSIS_SOURCE_NAME, - POLYGLOT_ANALYSIS_STAR_PROJECTIONS, - POLYGLOT_ANALYSIS_TABLE, - POLYGLOT_ANALYSIS_TRANSFORM_AGGREGATION, - POLYGLOT_ANALYSIS_TRANSFORM_CAST, - POLYGLOT_ANALYSIS_TRANSFORM_CONSTANT, - POLYGLOT_ANALYSIS_TRANSFORM_DIRECT, - POLYGLOT_ANALYSIS_TRANSFORM_KIND, - POLYGLOT_ANALYSIS_UPSTREAM, - POLYGLOT_PAYLOAD_COLUMN, -) -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event +from sqlbuild.compiler.lineage.types import PolyglotAnalysisDialect -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.lineage") _POLYGLOT_DIALECT_ALIASES: dict[str, PolyglotAnalysisDialect] = { "": PolyglotAnalysisDialect.GENERIC, "ansi": PolyglotAnalysisDialect.GENERIC, @@ -73,35 +29,13 @@ } -def build_rich_project_column_lineage( - *, - project: CompiledProject, - dialect: str | None = None, - model_names: frozenset[str] | None = None, -) -> ProjectColumnLineage | None: - """Build a rich project column lineage graph using Polyglot query analysis.""" - - if not project.settings.sql_analysis: - return None +def assemble_rich_project_column_lineage( + model_results: dict[str, ModelColumnLineage], +) -> ProjectColumnLineage: + """Collapse per-model rich lineage, in model order, into the project graph and its edges.""" - schema: dict[str, dict[str, str]] = _build_schema_mapping(project) - polyglot_tables: dict[str, dict[str, object]] = _polyglot_schema_tables(schema) - model_results: dict[str, ModelColumnLineage] = {} collapsed_edges: list[ColumnLineageEdge] = [] - - for model in project.models: - if model_names is not None and model.name not in model_names: - continue - result: ModelColumnLineage | None = _build_polyglot_model_column_lineage( - model=model, - schema=schema, - polyglot_tables=polyglot_tables, - dialect=dialect, - ) - if result is None: - continue - model_results[model.name] = result - target_resource_name: str = model.name + for target_resource_name, result in model_results.items(): for column in result.columns: target: QualifiedLineageColumn = QualifiedLineageColumn( resource_type=CompiledResourceType.MODEL, @@ -121,208 +55,6 @@ def build_rich_project_column_lineage( return ProjectColumnLineage(models=model_results, edges=tuple(collapsed_edges)) -def _build_polyglot_model_column_lineage( - *, - model: CompiledModel, - schema: dict[str, dict[str, str]], - polyglot_tables: dict[str, dict[str, object]], - dialect: str | None, -) -> ModelColumnLineage | None: - normalized_sql: str - physical_resources: tuple[PhysicalResource, ...] - normalized_sql, physical_resources = _normalize_sqlbuild_refs(model.query_sql) - resource_by_physical_name: dict[str, PhysicalResource] = { - resource.physical_name: resource for resource in physical_resources - } - polyglot_module: Any = import_polyglot_sql() - try: - analysis: Any = polyglot_module.analyze_query( - normalized_sql, - { - "dialect": _polyglot_dialect(dialect), - "schema": _referenced_polyglot_schema( - polyglot_tables=polyglot_tables, - physical_resources=physical_resources, - ), - }, - ) - except polyglot_module.PolyglotError as error: - log_debug_event( - logger=_DEBUG_LOGGER, - message="rich column lineage analysis failed; skipping model", - sqlbuild_model=model.name, - sqlbuild_error=str(error), - ) - return None - if not isinstance(analysis, dict): - return None - projections: object = analysis.get(POLYGLOT_ANALYSIS_PROJECTIONS) - if not isinstance(projections, list): - return None - - lineages: list[ColumnLineage] = [] - star_expanded_columns: frozenset[str] = _star_expanded_columns(analysis) - projection: object - for projection in projections: - if not isinstance(projection, dict): - continue - if bool(projection.get(POLYGLOT_ANALYSIS_IS_STAR)): - continue - output_value: object = projection.get(POLYGLOT_ANALYSIS_NAME) - output_column: str = output_value if isinstance(output_value, str) else "" - if not output_column: - continue - if output_column in star_expanded_columns: - continue - upstream_columns, confidence = _projection_upstreams( - projection=projection, - resource_by_physical_name=resource_by_physical_name, - ) - transform_kind: ColumnTransformKind = _projection_transform_kind( - projection=projection, - has_upstream=bool(upstream_columns), - ) - lineages.append( - ColumnLineage( - output_column=output_column, - transform_kind=transform_kind, - expression_sql=None, - upstream_columns=upstream_columns, - nullability=_projection_nullability(projection), - confidence=( - confidence - if upstream_columns or transform_kind == ColumnTransformKind.CONSTANT - else ColumnLineageConfidence.UNKNOWN - ), - ) - ) - - has_star: bool = bool(star_expanded_columns) - if has_star: - lineages.extend( - _build_star_lineage( - model=model, - schema=schema, - physical_resources=physical_resources, - existing_columns={lineage.output_column for lineage in lineages}, - ) - ) - - return ModelColumnLineage( - model_name=model.name, - columns=tuple(lineages), - has_star=has_star, - ) - - -def _projection_upstreams( - *, - projection: dict[str, Any], - resource_by_physical_name: dict[str, PhysicalResource], -) -> tuple[tuple[ColumnLineageSource, ...], ColumnLineageConfidence]: - upstream_values: object = projection.get(POLYGLOT_ANALYSIS_UPSTREAM) - if not isinstance(upstream_values, list): - return (), ColumnLineageConfidence.UNKNOWN - sources: list[ColumnLineageSource] = [] - seen: set[tuple[CompiledResourceType, str, str]] = set() - confidence: ColumnLineageConfidence = ColumnLineageConfidence.HIGH - upstream: object - for upstream in upstream_values: - if not isinstance(upstream, dict): - continue - column_value: object = upstream.get(POLYGLOT_PAYLOAD_COLUMN) - if ( - not isinstance(column_value, str) - or not column_value - or column_value == STAR_COLUMN_NAME - ): - continue - source_name: object = upstream.get(POLYGLOT_ANALYSIS_SOURCE_NAME) or upstream.get( - POLYGLOT_ANALYSIS_TABLE - ) - if not isinstance(source_name, str) or not source_name: - confidence = ColumnLineageConfidence.UNKNOWN - continue - resource: PhysicalResource | None = resource_by_physical_name.get(source_name) - if resource is None: - continue - source_confidence: object = upstream.get(POLYGLOT_ANALYSIS_SOURCE_CONFIDENCE) - source_alias: object = upstream.get(POLYGLOT_ANALYSIS_SOURCE_ALIAS) - if source_confidence != POLYGLOT_RESOLVED_SOURCE_CONFIDENCE and not isinstance( - source_alias, str - ): - confidence = ColumnLineageConfidence.MEDIUM - key: tuple[CompiledResourceType, str, str] = ( - resource.resource_type, - resource.resource_name, - column_value, - ) - if key in seen: - continue - seen.add(key) - sources.append( - ColumnLineageSource( - resource_type=resource.resource_type, - resource_name=resource.resource_name, - column_name=column_value, - ) - ) - return tuple( - sorted( - sources, - key=lambda source: ( - source.resource_type.value, - source.resource_name, - source.column_name, - ), - ) - ), confidence - - -def _star_expanded_columns(analysis: dict[str, Any]) -> frozenset[str]: - expanded: set[str] = set() - star_projections: object = analysis.get(POLYGLOT_ANALYSIS_STAR_PROJECTIONS) - if not isinstance(star_projections, list): - return frozenset() - star_projection: object - for star_projection in star_projections: - if not isinstance(star_projection, dict): - continue - columns: object = star_projection.get("expandedColumns") - if not isinstance(columns, list): - continue - column: object - for column in columns: - if isinstance(column, str) and column: - expanded.add(column) - return frozenset(expanded) - - -def _polyglot_schema_tables(schema: dict[str, dict[str, str]]) -> dict[str, dict[str, object]]: - tables: dict[str, dict[str, object]] = {} - for table_name, columns in schema.items(): - table_columns: list[dict[str, str]] = [] - for column_name, column_type in sorted(columns.items()): - table_columns.append({"name": column_name, "type": column_type or "UNKNOWN"}) - tables[table_name] = {"name": table_name, "columns": table_columns} - return tables - - -def _referenced_polyglot_schema( - *, - polyglot_tables: dict[str, dict[str, object]], - physical_resources: tuple[PhysicalResource, ...], -) -> dict[str, object]: - referenced: set[str] = {resource.physical_name for resource in physical_resources} - return { - "tables": [ - polyglot_tables[table_name] - for table_name in sorted(referenced) - if table_name in polyglot_tables - ] - } - - def _polyglot_dialect(dialect: str | TypeDialect | None) -> PolyglotAnalysisDialect: if dialect is None: return PolyglotAnalysisDialect.GENERIC @@ -331,27 +63,3 @@ def _polyglot_dialect(dialect: str | TypeDialect | None) -> PolyglotAnalysisDial if mapped is not None: return mapped return PolyglotAnalysisDialect(normalized) - - -def _projection_nullability(projection: dict[str, Any]) -> InferredNullability: - value: object = projection.get(POLYGLOT_ANALYSIS_NULLABILITY) - if value == POLYGLOT_ANALYSIS_NULLABILITY_NON_NULL: - return InferredNullability.NON_NULL - if value == POLYGLOT_ANALYSIS_NULLABILITY_NULLABLE: - return InferredNullability.NULLABLE - return InferredNullability.UNKNOWN - - -def _projection_transform_kind( - *, projection: dict[str, Any], has_upstream: bool -) -> ColumnTransformKind: - transform_kind: str = str(projection.get(POLYGLOT_ANALYSIS_TRANSFORM_KIND) or "") - if transform_kind == POLYGLOT_ANALYSIS_TRANSFORM_CAST: - return ColumnTransformKind.CAST - if transform_kind == POLYGLOT_ANALYSIS_TRANSFORM_AGGREGATION: - return ColumnTransformKind.AGGREGATION - if transform_kind == POLYGLOT_ANALYSIS_TRANSFORM_CONSTANT or not has_upstream: - return ColumnTransformKind.CONSTANT - if transform_kind == POLYGLOT_ANALYSIS_TRANSFORM_DIRECT: - return ColumnTransformKind.DIRECT - return ColumnTransformKind.EXPRESSION diff --git a/src/sqlbuild/compiler/lineage/_helpers/semantic_uses.py b/src/sqlbuild/compiler/lineage/_helpers/semantic_uses.py deleted file mode 100644 index 08d93b56e8..0000000000 --- a/src/sqlbuild/compiler/lineage/_helpers/semantic_uses.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Native direct semantic-use extraction helpers.""" - -from __future__ import annotations - -import json -import re -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import CompiledModel -from sqlbuild.compiler.lineage._helpers.columns import _normalize_sqlbuild_refs -from sqlbuild.compiler.lineage.exceptions import LineageAnalysisError -from sqlbuild.compiler.lineage.models import ( - DirectSemanticColumnUse, - PhysicalResource, - QualifiedLineageColumn, -) -from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, NativeUsageModule - - -def get_model_semantic_uses( # noqa: FFS002 - one native request/response translation boundary - *, model: CompiledModel, schema: dict[str, dict[str, str]], dialect: str | None -) -> tuple[DirectSemanticColumnUse, ...]: - normalized_sql, physical_resources = _normalize_sqlbuild_refs(model.query_sql) - resource_by_physical_name: dict[str, PhysicalResource] = { - resource.physical_name: resource for resource in physical_resources - } - tables: list[dict[str, object]] = [] - for name, columns in sorted(schema.items()): - table_columns: list[dict[str, str]] = [] - for column_name, column_type in columns.items(): - table_columns.append({"name": column_name, "type": column_type}) - tables.append({"name": name, "columns": table_columns}) - request: dict[str, object] = { - "sql": normalized_sql, - "dialect": dialect or "generic", - "schema": { - "strict": True, - "tables": tables, - }, - } - response: Any = json.loads( - cast(NativeUsageModule, _native).analyze_sql_uses_json( - json.dumps(request, separators=(",", ":"), sort_keys=True) - ) - ) - raw_uses: object = response.get("uses") if isinstance(response, dict) else None - if not isinstance(raw_uses, list): - raise LineageAnalysisError("native semantic usage analysis returned an invalid response") - result: list[DirectSemanticColumnUse] = [] - for raw_use in raw_uses: - if not isinstance(raw_use, dict): - continue - source_name: object = raw_use.get("sourceName") - column_name: object = raw_use.get("column") - context: object = raw_use.get("context") - expression_sql: object = raw_use.get("expressionSql") - if not all( - isinstance(value, str) for value in (source_name, column_name, context, expression_sql) - ): - continue - resource: PhysicalResource | None = resource_by_physical_name.get(str(source_name)) - if resource is None: - continue - line: int | None - column: int | None - line, column = _authored_position( - authored_sql=model.authored_sql, - column_name=str(column_name), - fallback_line=_optional_int(raw_use.get("line")), - fallback_column=_optional_int(raw_use.get("columnOffset")), - ) - result.append( - DirectSemanticColumnUse( - consumer_model=model.name, - context=str(context), - expression_sql=str(expression_sql), - source=QualifiedLineageColumn( - resource_type=resource.resource_type, - resource_name=resource.resource_name, - column_name=str(column_name), - ), - confidence=ColumnLineageConfidence(str(raw_use.get("confidence") or "unknown")), - line=line, - column=column, - ) - ) - return tuple(result) - - -def _optional_int(value: object) -> int | None: - return value if isinstance(value, int) else None - - -def _authored_position( - *, - authored_sql: str, - column_name: str, - fallback_line: int | None, - fallback_column: int | None, -) -> tuple[int | None, int | None]: - occurrences: list[re.Match[str]] = list( - re.finditer( - rf"(? ProjectColumnLineage | None: """Return the fast lineage graph, or None without SQL analysis.""" diff --git a/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py b/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py index 763b4354e0..07da567097 100644 --- a/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py +++ b/src/sqlbuild/compiler/lineage/main/build_semantic_uses.py @@ -2,21 +2,40 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.lineage._helpers.columns import _build_schema_mapping -from sqlbuild.compiler.lineage._helpers.semantic_uses import get_model_semantic_uses -from sqlbuild.compiler.lineage.models import DirectSemanticColumnUse +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.lineage._helpers.native_rich_columns import _schema_resources +from sqlbuild.compiler.lineage.models import DirectSemanticColumnUse, QualifiedLineageColumn +from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, NativeSemanticUseRow def build_direct_semantic_uses( - *, project: CompiledProject, dialect: str | None, model_names: frozenset[str] + *, project: CompiledProjectView, dialect: str | None, model_names: frozenset[str] ) -> tuple[DirectSemanticColumnUse, ...]: """Extract semantic uses only for explicitly requested models.""" - schema: dict[str, dict[str, str]] = _build_schema_mapping(project) - uses: list[DirectSemanticColumnUse] = [] - for model in project.models: - if model.name not in model_names: - continue - uses.extend(get_model_semantic_uses(model=model, schema=schema, dialect=dialect)) - return tuple(uses) + rows: list[NativeSemanticUseRow] = _native.build_direct_semantic_uses( + _schema_resources(project), + [ + (model.name, model.query_sql, model.authored_sql) + for model in project.models + if model.name in model_names + ], + dialect or "generic", + ) + return tuple( + DirectSemanticColumnUse( + consumer_model=model, + context=context, + expression_sql=expression, + source=QualifiedLineageColumn( + resource_type=CompiledResourceType(kind), + resource_name=resource, + column_name=column, + ), + confidence=ColumnLineageConfidence(confidence), + line=line, + column=offset, + ) + for model, context, expression, kind, resource, column, confidence, line, offset in rows + ) diff --git a/src/sqlbuild/compiler/lineage/main/columns.py b/src/sqlbuild/compiler/lineage/main/columns.py index 5a421f3e3f..1de9498177 100644 --- a/src/sqlbuild/compiler/lineage/main/columns.py +++ b/src/sqlbuild/compiler/lineage/main/columns.py @@ -2,9 +2,9 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.lineage._helpers.rich_columns import ( - build_rich_project_column_lineage, +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.lineage._helpers.native_rich_columns import ( + build_native_rich_project_column_lineage, ) from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( build_native_column_lineage, @@ -15,7 +15,7 @@ def build_project_column_lineage( *, - project: CompiledProject, + project: CompiledProjectView, dialect: str | None = None, mode: ColumnLineageMode = ColumnLineageMode.RICH, model_names: frozenset[str] | None = None, @@ -28,8 +28,6 @@ def build_project_column_lineage( project=project, dialect=dialect, model_names=model_names ) case ColumnLineageMode.RICH: - return build_rich_project_column_lineage( - project=project, - dialect=dialect, - model_names=model_names, + return build_native_rich_project_column_lineage( + project=project, dialect=dialect, model_names=model_names ) diff --git a/src/sqlbuild/compiler/lineage/types.py b/src/sqlbuild/compiler/lineage/types.py index 4708bfd72a..db075949c4 100644 --- a/src/sqlbuild/compiler/lineage/types.py +++ b/src/sqlbuild/compiler/lineage/types.py @@ -3,13 +3,8 @@ from __future__ import annotations from enum import StrEnum -from typing import Protocol - -class NativeUsageModule(Protocol): - """Native semantic-use analysis boundary.""" - - def analyze_sql_uses_json(self, request_json: str) -> str: ... +type NativeSemanticUseRow = tuple[str, str, str, str, str, str, str, int | None, int | None] class ColumnTransformKind(StrEnum): diff --git a/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py b/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py index 35a6b520ad..565800f529 100644 --- a/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py +++ b/src/sqlbuild/compiler/macro_bridge/_helpers/store_keys.py @@ -19,7 +19,6 @@ from sqlbuild.compiler.macro_bridge.constants import VALUE_RENDERER_FIELD from sqlbuild.compiler.macro_bridge.exceptions import UncacheableValueError from sqlbuild.compiler.scopes.models import DeclarationIdentity -from sqlbuild.python_nodes.models import SqlResourceRef _UNSTABLE_KEY_ERRORS: tuple[type[Exception], ...] = ( UncacheableValueError, @@ -73,30 +72,6 @@ def context_store_token( return None -def call_class_store_text( - *, - macro_tokens: tuple[str, ...], - context_token: str | None, - prior_relations: tuple[SqlResourceRef, ...] | None, -) -> str | None: - """Text of one persistent call class, or None when a relation name has no UTF-8 text.""" - - try: - return _encodable( - _key_text( - [ - list(macro_tokens), - context_token, - None - if prior_relations is None - else [[relation.kind.value, relation.name] for relation in prior_relations], - ] - ) - ) - except UnicodeEncodeError: - return None - - def _canonical_context(macro_context: MacroContext) -> object: fields: list[object] = [] for field in dataclasses.fields(macro_context): diff --git a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py index 0351f995f6..1ffb6d036c 100644 --- a/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py +++ b/src/sqlbuild/compiler/macro_bridge/classes/macro_bridge.py @@ -4,16 +4,9 @@ import logging import sys -from collections.abc import Hashable, Mapping from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - ExpansionSpan, - LoadedMacro, - MacroContext, -) from sqlbuild.compiler.macro_bridge._helpers.store_environment import ( digest_module_files, module_digests_metadata, @@ -22,24 +15,10 @@ store_environment, unchanged_module_digests, ) -from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( - call_class_store_text, - context_store_token, - macro_store_token, -) from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME -from sqlbuild.compiler.macro_bridge.main._scan_macro_calls import scan_macro_calls -from sqlbuild.compiler.macro_bridge.main._splice_macro_calls import splice_macro_calls from sqlbuild.compiler.macro_bridge.models import ( - MacroCallClass, - MacroCallScan, - MacroCallSite, ModuleSources, ) -from sqlbuild.compiler.macro_bridge.types import MacroCallEvent, MacroCallRecord -from sqlbuild.compiler.scopes.models import DeclarationIdentity -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind class MacroBridge: @@ -48,11 +27,7 @@ class MacroBridge: def __init__(self, *, python_version: tuple[int, int], unicode_version: str) -> None: self._python_version: tuple[int, int] = python_version self._unicode_version: str = unicode_version - self._memo: _native.MacroCallMemo = _native.MacroCallMemo() - self._class_ids: dict[Hashable, int] = {} - self._retained: dict[int, object] = {} - self._name_tokens: dict[tuple[int, str], tuple[Hashable, str | None] | None] = {} - self._context_tokens: dict[tuple[int, int], tuple[Hashable, str | None]] = {} + self.memo: _native.MacroCallMemo = _native.MacroCallMemo() self._store_path: Path | None = None self._store_trusted: bool = False self._module_digests: dict[str, str] = {} @@ -68,12 +43,12 @@ def attach_store(self, *, cache_dir: Path, project_dir: Path, model_paths: list[ metadata: bytes loaded: int try: - if not self._observe_modules(): + if not self.observe_modules(): logging.getLogger(__name__).debug("Macro call store not used: unknown module code") return fingerprint: str = project_fingerprint(project_dir=project_dir, model_paths=model_paths) environment: str = store_environment(project_dir=project_dir, fingerprint=fingerprint) - metadata, loaded = self._memo.attach_store(str(path), environment) + metadata, loaded = self.memo.attach_store(str(path), environment) except (OSError, RuntimeError) as error: logging.getLogger(__name__).debug("Macro call store not loaded: %s", error) return @@ -83,14 +58,14 @@ def attach_store(self, *, cache_dir: Path, project_dir: Path, model_paths: list[ else {} ) if carried is None: - self._memo.discard_store() + self.memo.discard_store() carried = {} self._store_path = path self._store_trusted = True self._module_digests = {**carried, **self._module_digests} self._project_inputs = (project_dir, model_paths, fingerprint) - self._name_tokens.clear() - self._context_tokens.clear() + self.memo.reset_call_classes(True) + self._share_observed_modules() def save_store(self) -> None: """Save the results this compile used or recorded, unless code or files it read changed.""" @@ -100,7 +75,7 @@ def save_store(self) -> None: project_dir, model_paths, fingerprint = self._project_inputs try: if ( - not self._observe_modules() + not self.observe_modules() or digest_module_files(self._module_digests) != self._module_digests or project_fingerprint(project_dir=project_dir, model_paths=model_paths) != fingerprint @@ -109,13 +84,13 @@ def save_store(self) -> None: "Macro call store not saved: code or files changed" ) return - _ = self._memo.save_store( + _ = self.memo.save_store( str(self._store_path), module_digests_metadata(self._module_digests) ) except (OSError, RuntimeError) as error: logging.getLogger(__name__).debug("Macro call store not saved: %s", error) - def _observe_modules(self) -> bool: + def observe_modules(self) -> bool: """Digest the files of modules loaded since the last look; False when one is unknown.""" loaded: dict[str, object] = dict(sys.modules) @@ -133,134 +108,21 @@ def _observe_modules(self) -> bool: self._module_digests.get(path, digest) != digest for path, digest in digests.items() ): self._store_trusted = False + self._share_observed_modules() return False self._module_digests.update(digests) + self._share_observed_modules() return True - def scan(self, sql: str) -> MacroCallScan: - """The complete top-level call sites of `sql`, and where its scan raises, if it does.""" - - return scan_macro_calls( - sql=sql, python_version=self._python_version, unicode_version=self._unicode_version - ) - - def call_class( - self, - *, - site: MacroCallSite, - declarations: DeclarationResolutionContext | None, - loaded_macros: dict[str, LoadedMacro], - macro_context: MacroContext, - ) -> MacroCallClass | None: - """Return what a call's result may depend on besides its text, or None if unresolvable.""" - - if site.tree_names is None: - return None - owner: object = loaded_macros if declarations is None else declarations - macros: Mapping[str, LoadedMacro] = ( - loaded_macros if declarations is None else declarations.macros - ) - _ = self._retained.setdefault(id(owner), owner) - _ = self._retained.setdefault(id(macro_context), macro_context) - tokens: list[Hashable] = [] - store_tokens: list[str | None] = [] - for name in site.tree_names: - token: tuple[Hashable, str | None] | None = self._name_token( - owner=owner, macros=macros, declarations=declarations, name=name - ) - if token is None: - return None - tokens.append(token[0]) - store_tokens.append(token[1]) - injects_context: bool = any(macros[name].injects_context for name in site.tree_names) - context: tuple[Hashable, str | None] | None = ( - self._context_token(owner=owner, declarations=declarations, macro_context=macro_context) - if injects_context - else None - ) - persistent_tokens: tuple[str, ...] = tuple( - token for token in store_tokens if token is not None - ) - return MacroCallClass( - key=(tuple(tokens), None if context is None else context[0]), - macro_store_tokens=persistent_tokens, - context_store_token=None if context is None else context[1], - persistent=len(persistent_tokens) == len(store_tokens) - and (context is None or context[1] is not None), - ) + def _share_observed_modules(self) -> None: + """Tell the memo how many modules a trusted store has seen, so it notices new imports.""" - def lookup( - self, - *, - call_class: MacroCallClass, - call_text: str, - prior_relations: tuple[SqlResourceRef, ...] | None, - ) -> tuple[int, MacroCallRecord | None]: - """Return the class id of a call and its recorded or stored result, if any.""" - - class_key: Hashable = (call_class.key, prior_relations) - class_id: int | None = self._class_ids.get(class_key) - if class_id is None: - class_id = self._class_ids[class_key] = len(self._class_ids) - class_text: str | None = ( - call_class_store_text( - macro_tokens=call_class.macro_store_tokens, - context_token=call_class.context_store_token, - prior_relations=prior_relations, - ) - if self._store_path is not None and call_class.persistent - else None - ) - if class_text is not None: - self._memo.set_persistent_class(class_id, class_text) - found: tuple[str, list[tuple[str, str]], list[tuple[int, str, str]]] | None = ( - self._memo.lookup(class_id, call_text) - ) - if found is None: - return class_id, None - sql, relations, events = found - return class_id, ( - sql, - tuple( - SqlResourceRef(kind=SqlResourceRefKind(kind), name=name) for kind, name in relations - ), - tuple(events), - ) - - def record( - self, - *, - class_id: int, - call_text: str, - sql: str, - relations: tuple[SqlResourceRef, ...], - events: tuple[MacroCallEvent, ...], - ) -> None: - """Record the result of one executed call for later calls of the same class and text.""" - - if self._store_trusted and len(sys.modules) != len(self._observed_modules): - _ = self._observe_modules() - self._memo.record( - class_id, - call_text, - ( - sql, - [(relation.kind.value, relation.name) for relation in relations], - list(events), - ), - ) - - def splice( - self, *, sql: str, sites: tuple[MacroCallSite, ...], outputs: list[str] - ) -> tuple[str, tuple[ExpansionSpan, ...]]: - """Replace each call site with its output, returning the SQL and substitution spans.""" - - return splice_macro_calls(sql=sql, sites=sites, outputs=outputs) + self.memo.observe_modules(len(self._observed_modules) if self._store_trusted else None) def stats(self) -> tuple[int, int, int]: """Memo hits, misses and recorded results so far.""" - return self._memo.stats() + return self.memo.stats() def store_module_paths(self) -> tuple[str, ...]: """Module files the stored call results depend on, imported this compile or not.""" @@ -270,59 +132,4 @@ def store_module_paths(self) -> tuple[str, ...]: def store_stats(self) -> tuple[int, int]: """Calls found in the store and calls recorded into it.""" - return self._memo.store_stats() - - def _name_token( - self, - *, - owner: object, - macros: Mapping[str, LoadedMacro], - declarations: DeclarationResolutionContext | None, - name: str, - ) -> tuple[Hashable, str | None] | None: - key: tuple[int, str] = (id(owner), name) - if key in self._name_tokens: - return self._name_tokens[key] - loaded: LoadedMacro | None = macros.get(name) - token: tuple[Hashable, str | None] | None = None - if loaded is not None and ( - declarations is None or name not in declarations.inaccessible_macros - ): - identity: DeclarationIdentity | None = ( - declarations.macro_records[name].identity if declarations is not None else None - ) - token = ( - (name, id(loaded), identity), - macro_store_token(loaded=loaded, identity=identity) - if self._store_path is not None - else None, - ) - self._name_tokens[key] = token - return token - - def _context_token( - self, - *, - owner: object, - declarations: DeclarationResolutionContext | None, - macro_context: MacroContext, - ) -> tuple[Hashable, str | None]: - key: tuple[int, int] = (id(owner), id(macro_context)) - token: tuple[Hashable, str | None] | None = self._context_tokens.get(key) - if token is None: - token = ( - ( - id(macro_context), - None - if declarations is None - else ( - tuple((name, id(value)) for name, value in declarations.constants.items()), - tuple((name, id(value)) for name, value in declarations.enums.items()), - ), - ), - context_store_token(macro_context=macro_context, declarations=declarations) - if self._store_path is not None - else None, - ) - self._context_tokens[key] = token - return token + return self.memo.store_stats() diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index 89f27cde8e..5ab8dee8e8 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -13,10 +13,6 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR -MACRO_USE_EVENT: int = 0 -DECLARATION_READ_EVENT: int = 1 -GENERATED_SQL_EVENT: int = 2 -ARGUMENT_REFERENCE_EVENT: int = 3 GENERATED_REFERENCE_MARKERS: tuple[str, ...] = tuple( f"{kind.function_name}(" for kind in SqlReferenceKind ) diff --git a/src/sqlbuild/compiler/macro_bridge/main/_macro_store_tokens.py b/src/sqlbuild/compiler/macro_bridge/main/_macro_store_tokens.py new file mode 100644 index 0000000000..a8e9e0379b --- /dev/null +++ b/src/sqlbuild/compiler/macro_bridge/main/_macro_store_tokens.py @@ -0,0 +1,14 @@ +"""The value text of a stored macro call's inputs, as the native memo keys persistent classes.""" + +from __future__ import annotations + +from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( + context_store_token, + macro_store_token, +) + + +def macro_store_tokens() -> dict[str, object]: + """`macro_store_token` and `context_store_token`, by the name the native memo reads them.""" + + return {"macro_token": macro_store_token, "context_token": context_store_token} diff --git a/src/sqlbuild/compiler/macro_bridge/main/_scan_macro_calls.py b/src/sqlbuild/compiler/macro_bridge/main/_scan_macro_calls.py deleted file mode 100644 index 477f3a12e7..0000000000 --- a/src/sqlbuild/compiler/macro_bridge/main/_scan_macro_calls.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Scan one authored string's top-level macro call sites natively.""" - -from __future__ import annotations - -import sys -import unicodedata - -import sqlbuild._native as _native -from sqlbuild.compiler.macro_bridge.models import MacroCallScan, MacroCallSite, MacroScanFailure - - -def scan_macro_calls( - *, - sql: str, - python_version: tuple[int, int] | None = None, - unicode_version: str | None = None, -) -> MacroCallScan: - """The complete top-level call sites of `sql`, and where its scan raises, if it does.""" - - rows: list[tuple[int, int, int, str, list[str] | None, bool]] - failure: tuple[int | None, str | None, str] | None - rows, failure = _native.scan_macro_call_sites( - sql, - python_version or (sys.version_info[0], sys.version_info[1]), - unicode_version or unicodedata.unidata_version, - ) - return MacroCallScan( - sites=tuple( - MacroCallSite( - start=start, - args_start=args_start, - end=end, - name=name, - tree_names=None if tree_names is None else tuple(tree_names), - typed_reference_text=typed_reference_text, - ) - for start, args_start, end, name, tree_names, typed_reference_text in rows - ), - failure=None - if failure is None - else MacroScanFailure(call_start=failure[0], call_name=failure[1], message=failure[2]), - ) diff --git a/src/sqlbuild/compiler/macro_bridge/main/_splice_macro_calls.py b/src/sqlbuild/compiler/macro_bridge/main/_splice_macro_calls.py deleted file mode 100644 index 480f4ef956..0000000000 --- a/src/sqlbuild/compiler/macro_bridge/main/_splice_macro_calls.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Splice macro call outputs into one authored string natively.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ExpansionSpan -from sqlbuild.compiler.macro_bridge.models import MacroCallSite - - -def splice_macro_calls( - *, sql: str, sites: tuple[MacroCallSite, ...], outputs: list[str] -) -> tuple[str, tuple[ExpansionSpan, ...]]: - """Replace each call site with its output, returning the SQL and substitution spans.""" - - rendered: str - spans: list[tuple[int, int, int, int]] - rendered, spans = _native.splice_macro_calls( - sql, [(site.start, site.end) for site in sites], outputs - ) - return rendered, tuple( - ExpansionSpan( - source_start=source_start, - source_end=source_end, - output_start=output_start, - output_end=output_end, - ) - for source_start, source_end, output_start, output_end in spans - ) diff --git a/src/sqlbuild/compiler/macro_bridge/models.py b/src/sqlbuild/compiler/macro_bridge/models.py index 657fdf907d..6b7c5a9be1 100644 --- a/src/sqlbuild/compiler/macro_bridge/models.py +++ b/src/sqlbuild/compiler/macro_bridge/models.py @@ -1,51 +1,10 @@ -"""Macro call sites found by the native scan and the classes of their results.""" +"""Module sources a stored macro call result depends on.""" from __future__ import annotations -from collections.abc import Hashable from dataclasses import dataclass -@dataclass(frozen=True) -class MacroCallSite: - """One top-level macro call in an authored string, in Python string offsets.""" - - start: int - args_start: int - end: int - name: str - tree_names: tuple[str, ...] | None - """Every macro name of the call; None where a nested call is not one the scan completes.""" - typed_reference_text: bool - - -@dataclass(frozen=True) -class MacroScanFailure: - """Where Python's scan of a string raises: in the call at `call_start`, or between calls.""" - - call_start: int | None - call_name: str | None - message: str - - -@dataclass(frozen=True) -class MacroCallScan: - """A string's complete top-level call sites in order, and where its scan raises, if it does.""" - - sites: tuple[MacroCallSite, ...] - failure: MacroScanFailure | None - - -@dataclass(frozen=True) -class MacroCallClass: - """A call's inputs besides its text: by object identity for the memo, by value for the store.""" - - key: Hashable - macro_store_tokens: tuple[str, ...] - context_store_token: str | None - persistent: bool - - @dataclass(frozen=True) class ModuleSources: """Files backing loaded modules, and whether every module's backing could be identified.""" diff --git a/src/sqlbuild/compiler/macro_bridge/types.py b/src/sqlbuild/compiler/macro_bridge/types.py deleted file mode 100644 index 767a96c96c..0000000000 --- a/src/sqlbuild/compiler/macro_bridge/types.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Macro bridge type aliases.""" - -from __future__ import annotations - -from sqlbuild.python_nodes.models import SqlResourceRef - -type MacroCallEvent = tuple[int, str, str] -type MacroCallRecord = tuple[str, tuple[SqlResourceRef, ...], tuple[MacroCallEvent, ...]] diff --git a/src/sqlbuild/compiler/manifest/_helpers/graph_maps.py b/src/sqlbuild/compiler/manifest/_helpers/graph_maps.py deleted file mode 100644 index 0d10337917..0000000000 --- a/src/sqlbuild/compiler/manifest/_helpers/graph_maps.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Parent/child map construction for manifest graph relationships.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, -) -from sqlbuild.compiler.manifest._helpers.shared import _key_to_unique_id - - -def build_unique_id(*, key: CompiledObjectKey, project_name: str, project: CompiledProject) -> str: - """Build a dbt-style unique_id from a CompiledObjectKey.""" - del project - return _key_to_unique_id(key=key, project_name=project_name) - - -def build_parent_map( - *, - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - project_name: str, - project: CompiledProject, -) -> dict[str, list[str]]: - """Build the parent_map (node -> first-order parents).""" - - result: dict[str, list[str]] = {} - key: CompiledObjectKey - parents: tuple[CompiledObjectKey, ...] - for key, parents in upstream_deps.items(): - node_id: str = build_unique_id(key=key, project_name=project_name, project=project) - result[node_id] = [ - build_unique_id(key=parent, project_name=project_name, project=project) - for parent in parents - ] - return result - - -def build_child_map( - *, - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - project_name: str, - project: CompiledProject, -) -> dict[str, list[str]]: - """Build the child_map (node -> first-order children).""" - - result: dict[str, list[str]] = {} - key: CompiledObjectKey - children: tuple[CompiledObjectKey, ...] - for key, children in downstream_deps.items(): - node_id: str = build_unique_id(key=key, project_name=project_name, project=project) - result[node_id] = [ - build_unique_id(key=child, project_name=project_name, project=project) - for child in children - ] - return result diff --git a/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py b/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py index 19ef455999..095a59dc26 100644 --- a/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py +++ b/src/sqlbuild/compiler/manifest/_helpers/model_nodes.py @@ -4,7 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash @@ -20,7 +20,7 @@ def build_model_node( *, - model: CompiledModel, + model: CompiledModelView, plan_entry: ModelPlanEntry | None, project_name: str, python_hook_metadata: dict[str, dict[str, object]] | None = None, @@ -75,7 +75,7 @@ def build_model_node( } -def _extract_tags(model: CompiledModel) -> list[str]: +def _extract_tags(model: CompiledModelView) -> list[str]: """Extract tags from model config.""" raw: object | None = model.config.values.get("tags") @@ -86,7 +86,7 @@ def _extract_tags(model: CompiledModel) -> list[str]: return [] -def _extract_description(model: CompiledModel) -> str: +def _extract_description(model: CompiledModelView) -> str: """Extract description from schema entry.""" if model.schema_entry is not None and model.schema_entry.description is not None: @@ -95,7 +95,7 @@ def _extract_description(model: CompiledModel) -> str: def _extract_meta( - *, model: CompiledModel, python_hook_metadata: dict[str, dict[str, object]] + *, model: CompiledModelView, python_hook_metadata: dict[str, dict[str, object]] ) -> dict[str, object]: """Extract meta from schema entry.""" diff --git a/src/sqlbuild/compiler/manifest/_helpers/payload.py b/src/sqlbuild/compiler/manifest/_helpers/payload.py new file mode 100644 index 0000000000..e68790fa21 --- /dev/null +++ b/src/sqlbuild/compiler/manifest/_helpers/payload.py @@ -0,0 +1,166 @@ +"""Build a dbt-compatible manifest.json payload from compiled project state.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledSeedView, + CompiledSourceView, +) +from sqlbuild.compiler.manifest._helpers.macros import ( + build_manifest_macro_nodes, +) +from sqlbuild.compiler.manifest._helpers.model_nodes import build_model_node +from sqlbuild.compiler.manifest._helpers.seeds import build_seed_node +from sqlbuild.compiler.manifest._helpers.sources import build_source_node +from sqlbuild.compiler.manifest._helpers.tests import ( + build_audit_test_nodes, + build_sql_test_nodes, +) +from sqlbuild.compiler.manifest.constants import DBT_MANIFEST_SCHEMA_VERSION, RESOURCE_TYPE_PREFIX +from sqlbuild.compiler.planner.models import ( + AuditPlanEntry, + ModelPlanEntry, + PlanOutput, + SqlTestPlanEntry, +) +from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities +from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity + + +def build_manifest_payload( + *, + project: CompiledProjectView, + plan_output: PlanOutput | None, + project_name: str, + adapter_type: str, + graph: _native.NativeProjectGraph, +) -> dict[str, object]: + """A full dbt v12-compatible manifest; parent and child maps come from the native graph.""" + + model_plan_map: dict[str, ModelPlanEntry] = {} + source_load_names: frozenset[str] = frozenset() + if plan_output is not None: + model_plan_map = {entry.name: entry for entry in plan_output.model_entries} + source_load_names = frozenset(entry.name for entry in plan_output.source_load_entries) + + nodes: dict[str, dict[str, object]] = {} + sources: dict[str, dict[str, object]] = {} + macros: dict[str, dict[str, object]] = build_manifest_macro_nodes( + loaded_macros=project.loaded_macros, + sql_hook_files=project.sql_hook_files, + project_name=project_name, + scope_index=project.scope_index, + ) + python_hook_metadata: dict[str, dict[str, object]] = _build_python_hook_metadata(project) + + model: CompiledModelView + for model in project.models: + plan_entry: ModelPlanEntry | None = model_plan_map.get(model.name) + unique_id: str = f"model.{project_name}.{model.name}" + nodes[unique_id] = build_model_node( + model=model, + plan_entry=plan_entry, + project_name=project_name, + python_hook_metadata=python_hook_metadata, + sql_analysis_dialect=project.sql_analysis_dialect, + ) + + source: CompiledSourceView + for source in project.sources: + unique_id = f"source.{project_name}.{source.name}" + sources[unique_id] = build_source_node( + source=source, + project_name=project_name, + auto_load=source.name in source_load_names, + ) + + seed: CompiledSeedView + for seed in project.seeds: + unique_id = f"seed.{project_name}.{seed.name}" + nodes[unique_id] = build_seed_node( + seed=seed, + project_name=project_name, + ) + + if plan_output is not None: + audit_entry: AuditPlanEntry + for audit_entry in plan_output.audit_entries: + audit_nodes: dict[str, dict[str, object]] = build_audit_test_nodes( + audit_entry=audit_entry, + project_name=project_name, + ) + nodes.update(audit_nodes) + + test_entry: SqlTestPlanEntry + for test_entry in plan_output.test_entries: + test_nodes: dict[str, dict[str, object]] = build_sql_test_nodes( + test_entry=test_entry, + project_name=project_name, + ) + nodes.update(test_nodes) + + parent_rows, child_rows = graph.unique_id_maps(project_name, dict(RESOURCE_TYPE_PREFIX)) + parent_map: dict[str, list[str]] = dict(parent_rows) + child_map: dict[str, list[str]] = dict(child_rows) + + return { + "metadata": _build_metadata( + project_name=project_name, + adapter_type=adapter_type, + run_id=project.run_id, + ), + "nodes": nodes, + "sources": sources, + "macros": macros, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": parent_map, + "child_map": child_map, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {}, + } + + +def _build_python_hook_metadata(project: CompiledProjectView) -> dict[str, dict[str, object]]: + hook_identities: dict[str, PythonNodeIdentity] = build_hook_identities(project.hook_functions) + return { + hook.name: { + "relative_path": hook.relative_path.as_posix(), + "definition_hash": hook_identities[hook.name].definition_hash, + "version_hash": hook_identities[hook.name].version_hash, + } + for hook in project.hook_functions + } + + +def _build_metadata( + *, + project_name: str, + adapter_type: str, + run_id: str, +) -> dict[str, object]: + """Build the top-level manifest metadata block.""" + + return { + "dbt_schema_version": DBT_MANIFEST_SCHEMA_VERSION, + "dbt_version": "1.12.0", + "generated_at": datetime.now(tz=UTC).isoformat(), + "invocation_id": run_id, + "env": {}, + "project_name": project_name, + "project_id": None, + "user_id": None, + "send_anonymous_usage_stats": False, + "adapter_type": adapter_type, + } diff --git a/src/sqlbuild/compiler/manifest/_helpers/seeds.py b/src/sqlbuild/compiler/manifest/_helpers/seeds.py index ef63851070..abe698b282 100644 --- a/src/sqlbuild/compiler/manifest/_helpers/seeds.py +++ b/src/sqlbuild/compiler/manifest/_helpers/seeds.py @@ -2,14 +2,14 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledSeed +from sqlbuild.compiler.compile.types import CompiledSeedView from sqlbuild.compiler.manifest._helpers.shared import build_columns_dict, build_fqn from sqlbuild.compiler.manifest.constants import CHECKSUM_HASH_NAME, RESOURCE_TYPE_SEED def build_seed_node( *, - seed: CompiledSeed, + seed: CompiledSeedView, project_name: str, ) -> dict[str, object]: """Build one dbt-compatible seed node dict.""" diff --git a/src/sqlbuild/compiler/manifest/_helpers/shared.py b/src/sqlbuild/compiler/manifest/_helpers/shared.py index 6fe98e5523..d9bdd0aa70 100644 --- a/src/sqlbuild/compiler/manifest/_helpers/shared.py +++ b/src/sqlbuild/compiler/manifest/_helpers/shared.py @@ -5,8 +5,8 @@ from pathlib import Path from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import SqlHookEntry +from sqlbuild.compiler.manifest.constants import RESOURCE_TYPE_PREFIX from sqlbuild.spec.contracts.models import ( SchemaColumn, SchemaModelEntry, @@ -14,15 +14,6 @@ SourceColumnEntry, ) -_RESOURCE_TYPE_PREFIX: dict[str, str] = { - CompiledResourceType.MODEL: "model", - CompiledResourceType.SOURCE: "source", - CompiledResourceType.SEED: "seed", - CompiledResourceType.DBT_REF: "model", - CompiledResourceType.AUDIT: "test", - CompiledResourceType.SQL_TEST: "test", -} - def build_fqn(*, project_name: str, relative_path: Path) -> list[str]: """Build a dbt-style fully qualified name list from project name and path.""" @@ -150,7 +141,7 @@ def build_depends_on( def _key_to_unique_id(*, key: CompiledObjectKey, project_name: str) -> str: """Convert a CompiledObjectKey to a dbt-style unique_id.""" - resource_prefix: str = _RESOURCE_TYPE_PREFIX.get(str(key.resource_type), "model") + resource_prefix: str = RESOURCE_TYPE_PREFIX.get(str(key.resource_type), "model") return f"{resource_prefix}.{project_name}.{key.name}" diff --git a/src/sqlbuild/compiler/manifest/_helpers/sources.py b/src/sqlbuild/compiler/manifest/_helpers/sources.py index 4a10de6992..07c107a786 100644 --- a/src/sqlbuild/compiler/manifest/_helpers/sources.py +++ b/src/sqlbuild/compiler/manifest/_helpers/sources.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledSource +from sqlbuild.compiler.compile.types import CompiledSourceView from sqlbuild.compiler.manifest._helpers.shared import build_source_columns_dict from sqlbuild.compiler.manifest.constants import RESOURCE_TYPE_SOURCE from sqlbuild.spec.contracts.models import SourceEntry @@ -10,7 +10,7 @@ def build_source_node( *, - source: CompiledSource, + source: CompiledSourceView, project_name: str, auto_load: bool = False, ) -> dict[str, object]: diff --git a/src/sqlbuild/compiler/manifest/constants.py b/src/sqlbuild/compiler/manifest/constants.py index e94e6e6424..6c70425595 100644 --- a/src/sqlbuild/compiler/manifest/constants.py +++ b/src/sqlbuild/compiler/manifest/constants.py @@ -1,5 +1,7 @@ """Manifest output constants aligned with dbt manifest v12 schema.""" +from sqlbuild.compiler.compile.types import CompiledResourceType + DBT_MANIFEST_SCHEMA_VERSION: str = "https://schemas.getdbt.com/dbt/manifest/v12.json" RESOURCE_TYPE_MODEL: str = "model" @@ -9,3 +11,12 @@ RESOURCE_TYPE_MACRO: str = "macro" CHECKSUM_HASH_NAME: str = "sha256" + +RESOURCE_TYPE_PREFIX: dict[str, str] = { + CompiledResourceType.MODEL: "model", + CompiledResourceType.SOURCE: "source", + CompiledResourceType.SEED: "seed", + CompiledResourceType.DBT_REF: "model", + CompiledResourceType.AUDIT: "test", + CompiledResourceType.SQL_TEST: "test", +} diff --git a/src/sqlbuild/compiler/manifest/main/build.py b/src/sqlbuild/compiler/manifest/main/build.py index 82c0cd0240..913ee1df6c 100644 --- a/src/sqlbuild/compiler/manifest/main/build.py +++ b/src/sqlbuild/compiler/manifest/main/build.py @@ -2,177 +2,32 @@ from __future__ import annotations -from datetime import UTC, datetime - -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledSeed, - CompiledSource, -) -from sqlbuild.compiler.manifest._helpers.graph_maps import ( - build_child_map, - build_parent_map, -) -from sqlbuild.compiler.manifest._helpers.macros import ( - build_manifest_macro_nodes, -) -from sqlbuild.compiler.manifest._helpers.model_nodes import build_model_node -from sqlbuild.compiler.manifest._helpers.seeds import build_seed_node -from sqlbuild.compiler.manifest._helpers.sources import build_source_node -from sqlbuild.compiler.manifest._helpers.tests import ( - build_audit_test_nodes, - build_sql_test_nodes, -) -from sqlbuild.compiler.manifest.constants import DBT_MANIFEST_SCHEMA_VERSION -from sqlbuild.compiler.planner.models import ( - AuditPlanEntry, - ModelPlanEntry, - PlanOutput, - SqlTestPlanEntry, -) -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.graph.main._native_graph_from_edges import native_graph_from_edges +from sqlbuild.compiler.manifest._helpers.payload import build_manifest_payload +from sqlbuild.compiler.planner.models import PlanOutput def build_manifest( *, - project: CompiledProject, + project: CompiledProjectView, plan_output: PlanOutput | None = None, project_name: str, adapter_type: str, upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], ) -> dict[str, object]: - """Build a full dbt v12-compatible manifest dictionary.""" - - model_plan_map: dict[str, ModelPlanEntry] = {} - source_load_names: frozenset[str] = frozenset() - if plan_output is not None: - model_plan_map = {entry.name: entry for entry in plan_output.model_entries} - source_load_names = frozenset(entry.name for entry in plan_output.source_load_entries) + """Build a full dbt v12-compatible manifest dictionary from caller-built edge maps.""" - nodes: dict[str, dict[str, object]] = {} - sources: dict[str, dict[str, object]] = {} - macros: dict[str, dict[str, object]] = build_manifest_macro_nodes( - loaded_macros=project.loaded_macros, - sql_hook_files=project.sql_hook_files, - project_name=project_name, - scope_index=project.scope_index, + graph: _native.NativeProjectGraph = native_graph_from_edges( + upstream=upstream_deps, downstream=downstream_deps ) - python_hook_metadata: dict[str, dict[str, object]] = _build_python_hook_metadata(project) - - model: CompiledModel - for model in project.models: - plan_entry: ModelPlanEntry | None = model_plan_map.get(model.name) - unique_id: str = f"model.{project_name}.{model.name}" - nodes[unique_id] = build_model_node( - model=model, - plan_entry=plan_entry, - project_name=project_name, - python_hook_metadata=python_hook_metadata, - sql_analysis_dialect=project.sql_analysis_dialect, - ) - - source: CompiledSource - for source in project.sources: - unique_id = f"source.{project_name}.{source.name}" - sources[unique_id] = build_source_node( - source=source, - project_name=project_name, - auto_load=source.name in source_load_names, - ) - - seed: CompiledSeed - for seed in project.seeds: - unique_id = f"seed.{project_name}.{seed.name}" - nodes[unique_id] = build_seed_node( - seed=seed, - project_name=project_name, - ) - - if plan_output is not None: - audit_entry: AuditPlanEntry - for audit_entry in plan_output.audit_entries: - audit_nodes: dict[str, dict[str, object]] = build_audit_test_nodes( - audit_entry=audit_entry, - project_name=project_name, - ) - nodes.update(audit_nodes) - - test_entry: SqlTestPlanEntry - for test_entry in plan_output.test_entries: - test_nodes: dict[str, dict[str, object]] = build_sql_test_nodes( - test_entry=test_entry, - project_name=project_name, - ) - nodes.update(test_nodes) - - parent_map: dict[str, list[str]] = build_parent_map( - upstream_deps=upstream_deps, - project_name=project_name, + return build_manifest_payload( project=project, - ) - child_map: dict[str, list[str]] = build_child_map( - downstream_deps=downstream_deps, + plan_output=plan_output, project_name=project_name, - project=project, + adapter_type=adapter_type, + graph=graph, ) - - return { - "metadata": _build_metadata( - project_name=project_name, - adapter_type=adapter_type, - run_id=project.run_id, - ), - "nodes": nodes, - "sources": sources, - "macros": macros, - "exposures": {}, - "metrics": {}, - "groups": {}, - "selectors": {}, - "disabled": {}, - "parent_map": parent_map, - "child_map": child_map, - "group_map": {}, - "docs": {}, - "saved_queries": {}, - "semantic_models": {}, - "unit_tests": {}, - } - - -def _build_python_hook_metadata(project: CompiledProject) -> dict[str, dict[str, object]]: - hook_identities: dict[str, PythonNodeIdentity] = build_hook_identities(project.hook_functions) - return { - hook.name: { - "relative_path": hook.relative_path.as_posix(), - "definition_hash": hook_identities[hook.name].definition_hash, - "version_hash": hook_identities[hook.name].version_hash, - } - for hook in project.hook_functions - } - - -def _build_metadata( - *, - project_name: str, - adapter_type: str, - run_id: str, -) -> dict[str, object]: - """Build the top-level manifest metadata block.""" - - return { - "dbt_schema_version": DBT_MANIFEST_SCHEMA_VERSION, - "dbt_version": "1.12.0", - "generated_at": datetime.now(tz=UTC).isoformat(), - "invocation_id": run_id, - "env": {}, - "project_name": project_name, - "project_id": None, - "user_id": None, - "send_anonymous_usage_stats": False, - "adapter_type": adapter_type, - } diff --git a/src/sqlbuild/compiler/manifest/main/build_graph_manifest.py b/src/sqlbuild/compiler/manifest/main/build_graph_manifest.py new file mode 100644 index 0000000000..e43dec7fcb --- /dev/null +++ b/src/sqlbuild/compiler/manifest/main/build_graph_manifest.py @@ -0,0 +1,27 @@ +"""Build a dbt-compatible manifest.json from a compiled project and its native graph.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.manifest._helpers.payload import build_manifest_payload +from sqlbuild.compiler.planner.models import PlanOutput + + +def build_graph_manifest( + *, + project: CompiledProjectView, + plan_output: PlanOutput | None, + project_name: str, + adapter_type: str, + graph: _native.NativeProjectGraph, +) -> dict[str, object]: + """Build the manifest with parent and child maps read from the native lineage graph.""" + + return build_manifest_payload( + project=project, + plan_output=plan_output, + project_name=project_name, + adapter_type=adapter_type, + graph=graph, + ) diff --git a/src/sqlbuild/compiler/model_config/main/_native_contract_classes.py b/src/sqlbuild/compiler/model_config/main/_native_contract_classes.py new file mode 100644 index 0000000000..ee045084c6 --- /dev/null +++ b/src/sqlbuild/compiler/model_config/main/_native_contract_classes.py @@ -0,0 +1,29 @@ +"""Contract classes native header parsing builds Python columns and audits from.""" + +from functools import cache + +from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds +from sqlbuild.compiler.auditing.types import AuditSeverity, ThresholdOperator +from sqlbuild.spec.contracts.models import ( + SchemaAuditInstance, + SchemaColumn, + SchemaDynamicColumnFamily, + SourceLocation, +) + + +@cache +def native_contract_classes() -> dict[str, object]: + """The contract classes native header parsing builds columns and audits as.""" + + return { + "schema_column": SchemaColumn, + "schema_audit_instance": SchemaAuditInstance, + "schema_dynamic_column_family": SchemaDynamicColumnFamily, + "source_location": SourceLocation, + "measurement_thresholds": MeasurementThresholds, + "measurement_threshold_bound": MeasurementThresholdBound, + "severities": {severity.value: severity for severity in AuditSeverity}, + "threshold_operators": {operator.value: operator for operator in ThresholdOperator}, + "allocate": object.__new__, + } diff --git a/src/sqlbuild/compiler/model_config/main/_parse_native_header_metadata.py b/src/sqlbuild/compiler/model_config/main/_parse_native_header_metadata.py deleted file mode 100644 index 30fb1af692..0000000000 --- a/src/sqlbuild/compiler/model_config/main/_parse_native_header_metadata.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Parse a MODEL header's columns and audits natively.""" - -from __future__ import annotations - -from functools import cache -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds -from sqlbuild.compiler.auditing.types import AuditSeverity, ThresholdOperator -from sqlbuild.compiler.model_config.models import NativeHeaderMetadata -from sqlbuild.compiler.model_config.types import NativeHeaderMetadataRow -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaColumn, SourceLocation - - -def parse_native_header_metadata( - *, - raw_columns: object | None, - raw_audits: object | None, - column_locations: dict[str, SourceLocation], - file_path: Path, -) -> NativeHeaderMetadata: - """Return one model's header columns and audits, or the error each raises.""" - - row: NativeHeaderMetadataRow = _native.parse_model_header_metadata( - (raw_columns, raw_audits), (column_locations, str(file_path)), _contract_classes() - ) - columns, audits = row - return NativeHeaderMetadata( - columns=() if isinstance(columns, _native.NativeConfigError) else columns, - audits=() if audits is None or isinstance(audits, _native.NativeConfigError) else audits, - columns_error=columns if isinstance(columns, _native.NativeConfigError) else None, - audits_error=audits if isinstance(audits, _native.NativeConfigError) else None, - ) - - -@cache -def _contract_classes() -> dict[str, object]: - return { - "schema_column": SchemaColumn, - "schema_audit_instance": SchemaAuditInstance, - "measurement_thresholds": MeasurementThresholds, - "measurement_threshold_bound": MeasurementThresholdBound, - "severities": {severity.value: severity for severity in AuditSeverity}, - "threshold_operators": {operator.value: operator for operator in ThresholdOperator}, - "allocate": object.__new__, - } diff --git a/src/sqlbuild/compiler/model_config/models.py b/src/sqlbuild/compiler/model_config/models.py index 63b703aaca..1a95bb0fbd 100644 --- a/src/sqlbuild/compiler/model_config/models.py +++ b/src/sqlbuild/compiler/model_config/models.py @@ -5,17 +5,6 @@ from dataclasses import dataclass import sqlbuild._native as _native -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaColumn - - -@dataclass(frozen=True, slots=True) -class NativeHeaderMetadata: - """One model's header columns and audits, or the error parsing each raises.""" - - columns: tuple[SchemaColumn, ...] - audits: tuple[SchemaAuditInstance, ...] - columns_error: _native.NativeConfigError | None = None - audits_error: _native.NativeConfigError | None = None @dataclass(frozen=True, slots=True) diff --git a/src/sqlbuild/compiler/model_config/types.py b/src/sqlbuild/compiler/model_config/types.py deleted file mode 100644 index 7a0b52867a..0000000000 --- a/src/sqlbuild/compiler/model_config/types.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Shapes the native model config bindings return.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SchemaColumn - -type NativeHeaderMetadataRow = tuple[ - tuple[SchemaColumn, ...] | _native.NativeConfigError, - tuple[SchemaAuditInstance, ...] | _native.NativeConfigError | None, -] diff --git a/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py b/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py index 04592edf51..b77a6a61eb 100644 --- a/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py +++ b/src/sqlbuild/compiler/model_loop/main/_build_native_declaration_contexts.py @@ -11,13 +11,25 @@ ) from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration from sqlbuild.compiler.model_loop.constants import DECLARATION_KIND_CODES, NO_DECLARATION_KIND +from sqlbuild.compiler.scopes.exceptions import ( + InvalidQualifiedIdentityError, + InvalidScopePathError, +) from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationVisibilityIndex, + OwnershipRoot, + ResourceIdentity, + ResourceRecord, ScopeLookup, VisibilityRecord, ) -from sqlbuild.compiler.scopes.types import DeclarationKind, VisibilityReason +from sqlbuild.compiler.scopes.types import ( + DeclarationKind, + ResourceKind, + ScopeKind, + VisibilityReason, +) def build_native_declaration_contexts( @@ -59,11 +71,28 @@ def build_native_declaration_contexts( "allocate": object.__new__, "reasons": {reason.value: reason for reason in VisibilityReason}, }, - { - "identity_keys": identity_keys, - "private_positions": index.private_positions, - "grants_by_resource": lookup.grants_by_resource, - }, + ( + { + "identity_keys": identity_keys, + "private_positions": index.private_positions, + "grants_by_resource": lookup.grants_by_resource, + }, + { + "resources_by_path": lookup.resources_by_path, + "resources": lookup.resources, + "declarations": lookup.declarations, + "index_declarations": lookup.index.declarations, + "resource_record": ResourceRecord, + "ownership_root": OwnershipRoot, + "resource_identity": ResourceIdentity, + "declaration_identity": DeclarationIdentity, + "resource_kind": ResourceKind, + "declaration_kind": DeclarationKind, + "private_scope": ScopeKind.PRIVATE, + "identity_error": InvalidQualifiedIdentityError, + "path_error": InvalidScopePathError, + }, + ), ) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py index 1a5c79e100..452f64dafa 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py @@ -9,9 +9,8 @@ from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, CompiledObjectKey, - CompiledProject, - CompileProjectInputs, ) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineOptions from sqlbuild.compiler.planner.main.selection._resolve_planner_scopes import ( @@ -79,7 +78,7 @@ def resolve_configured_rule_selection( *, options: CompilePipelineOptions, discovered_inputs: DiscoveredProjectInputs, - project: CompiledProject, + project: CompiledProjectView, ) -> frozenset[CompiledObjectKey] | None: """Align configured-rule subjects with models that received deep SQL analysis.""" @@ -106,7 +105,7 @@ def resolve_configured_rule_selection( def resolve_analysis_model_names( *, - compile_inputs: CompileProjectInputs, + compile_inputs: CompileInputsView, inference_profile: ExpressionInferenceProfile, selection: CompileAnalysisSelection | None, ) -> frozenset[str] | None: @@ -114,7 +113,7 @@ def resolve_analysis_model_names( if selection is None or (not selection.select and not selection.exclude): return None - structural_project: CompiledProject = assemble_project( + structural_project: CompiledProjectView = assemble_project( inputs=compile_inputs, inference_profile=inference_profile, skip_column_inference=True, diff --git a/src/sqlbuild/compiler/pipeline/_helpers/clone.py b/src/sqlbuild/compiler/pipeline/_helpers/clone.py index 2fb7ef7cc5..5127f9eb66 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/clone.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/clone.py @@ -3,7 +3,8 @@ from __future__ import annotations from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompiledProject +from sqlbuild.compiler.compile.models import CompileAnalysisSelection +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.pipeline.models import ( @@ -27,7 +28,7 @@ def prepare_clone_pipeline( options: ClonePipelineOptions, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, ) -> ClonePipelineResult: - origin_project: CompiledProject = _compile_project_for_environment( + origin_project: CompiledProjectView = _compile_project_for_environment( discovered_inputs=discovered_inputs, adapter=adapter, target_name=origin_target_name, @@ -37,7 +38,7 @@ def prepare_clone_pipeline( cli_vars=options.cli_vars, external_sql_reference_resolver=external_sql_reference_resolver, ) - destination_project: CompiledProject = _compile_project_for_environment( + destination_project: CompiledProjectView = _compile_project_for_environment( discovered_inputs=discovered_inputs, adapter=adapter, target_name=destination_target_name, @@ -55,7 +56,7 @@ def prepare_clone_pipeline( discovered_inputs=discovered_inputs, target_name=destination_target_name, ) - origin_source_project: CompiledProject = ( + origin_source_project: CompiledProjectView = ( origin_project if origin_read_target_name == origin_target_name else _compile_project_for_environment( @@ -69,7 +70,7 @@ def prepare_clone_pipeline( external_sql_reference_resolver=external_sql_reference_resolver, ) ) - destination_source_project: CompiledProject = ( + destination_source_project: CompiledProjectView = ( destination_project if destination_read_target_name == destination_target_name else _compile_project_for_environment( @@ -126,7 +127,7 @@ def _compile_project_for_environment( no_cache: bool, cli_vars: dict[str, object] | None, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> CompiledProject: +) -> CompiledProjectView: return build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, diff --git a/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py b/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py index b1dddc1b06..af64844bfe 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py @@ -2,32 +2,27 @@ from __future__ import annotations -import re from collections.abc import Callable from typing import Any +import sqlbuild._native as _native from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile.constants import PRESERVE_TARGET_VALUE from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, CompiledRelationLocation, - CompiledSeed, ) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView, CompiledSeedView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.constants import DEFERRED_TARGET_CONTEXT_KEYS from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.spec.contracts.models import TargetConfig -_CTX_PATTERN: re.Pattern[str] = re.compile(r"\$\{CTX:([^}]+)\}") -_VAR_PATTERN: re.Pattern[str] = re.compile(r"\$\{([^}:]+)\}") - def build_deferred_locations( *, - project: CompiledProject, + project: CompiledProjectView, deferred_target_config: TargetConfig, effective_vars: dict[str, object], default_schema: str | None, @@ -37,7 +32,7 @@ def build_deferred_locations( """Build physical locations for all models and seeds under a deferred target.""" locations: dict[str, CompiledRelationLocation] = {} - model: CompiledModel + model: CompiledModelView for model in project.models: locations[model.name] = _resolve_deferred_location( target=model.destination, @@ -47,7 +42,7 @@ def build_deferred_locations( default_database=default_database, render_qualified_name=render_qualified_name, ) - seed: CompiledSeed + seed: CompiledSeedView for seed in project.seeds: locations[seed.name] = _resolve_deferred_location( target=seed.destination, @@ -155,25 +150,17 @@ def _resolve_target_field( if target_value is None or target_value == PRESERVE_TARGET_VALUE: return logical_value - result: str = target_value - - def _replace_ctx(match: re.Match[str]) -> str: - ctx_key: str = match.group(1) - if ctx_key in DEFERRED_TARGET_CONTEXT_KEYS: - return logical_value if logical_value is not None else "" - return match.group(0) - - result = _CTX_PATTERN.sub(_replace_ctx, result) - - def _replace_var(match: re.Match[str]) -> str: - var_name: str = match.group(1) - if var_name not in effective_vars: - return match.group(0) - return render_project_var_text( - value=effective_vars[var_name], - label=f"deferred target variable '${{{var_name}}}'", + segments: list[tuple[bool, str]] = _native.deferred_target_segments( + target_value, + "" if logical_value is None else logical_value, + sorted(DEFERRED_TARGET_CONTEXT_KEYS), + [name for name in effective_vars if isinstance(name, str)], + ) + return "".join( + render_project_var_text( + value=effective_vars[text], label=f"deferred target variable '${{{text}}}'" ) - - result = _VAR_PATTERN.sub(_replace_var, result) - - return result + if is_variable + else text + for is_variable, text in segments + ) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/diff.py b/src/sqlbuild/compiler/pipeline/_helpers/diff.py index 9edc690bd2..a2959bd7e5 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/diff.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/diff.py @@ -5,17 +5,12 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.graph.main._build_lineage_downstream_deps import ( - build_lineage_downstream_deps, -) -from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import ( - build_lineage_upstream_deps, -) +from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many +from sqlbuild.compiler.graph.models import LineageGraphViews from sqlbuild.compiler.pipeline.constants import ( MODEL_PATH_ROOT, PATH_SELECTOR_PREFIX, @@ -23,12 +18,6 @@ ) from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection._build_model_path_index import ( - build_model_path_index, -) -from sqlbuild.compiler.planner.main.selection._build_model_tag_index import ( - build_model_tag_index, -) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver @@ -41,7 +30,7 @@ def compile_project_for_diff_environment( no_sql_validation: bool, cli_vars: dict[str, object] | None = None, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """Compile a project for one diff environment.""" return build_compiled_project( @@ -57,18 +46,13 @@ def compile_project_for_diff_environment( def resolve_diff_model_names( *, - project: CompiledProject, + project: CompiledProjectView, select: tuple[str, ...], exclude: tuple[str, ...], ) -> tuple[str, ...]: """Resolve diff selectors to model names only.""" - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_upstream_deps(project) - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_downstream_deps(upstream_deps) - ) + lineage: LineageGraphViews = project_lineage_views(project) all_keys: dict[str, CompiledObjectKey] = { **{model.name: model.key for model in project.models}, **{source.name: source.key for source in project.sources}, @@ -78,10 +62,10 @@ def resolve_diff_model_names( select=select, exclude=exclude, all_keys=all_keys, - upstream=upstream_deps, - downstream=downstream_deps, - tag_index=build_model_tag_index(project), - path_index=build_model_path_index(project), + upstream=lineage.upstream_deps, + downstream=lineage.downstream_deps, + tag_index=lineage.tag_index, + path_index=lineage.path_index, ) return tuple( model.name diff --git a/src/sqlbuild/compiler/pipeline/_helpers/graph.py b/src/sqlbuild/compiler/pipeline/_helpers/graph.py index a95c4e585a..21ce3d6f05 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/graph.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/graph.py @@ -2,22 +2,21 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, -) - - -def build_static_all_keys(project: CompiledProject) -> dict[str, CompiledObjectKey]: - """Build selector lookup keys for all named graph resources.""" - - keys: dict[str, CompiledObjectKey] = {} - for model in project.models: - keys[model.name] = model.key - for source in project.sources: - keys[source.name] = source.key - for seed in project.seeds: - keys[seed.name] = seed.key - for function in project.functions: - keys[function.name] = function.key - return keys +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.compiler.planner.main.selection._graph_selection import resolve_graph_selection + + +def build_project_graph_impl(project: CompiledProjectView) -> ProjectGraph: + """The project's natively held graph; dict views are built only when a boundary reads them.""" + + return ProjectGraph(project=project, native=project.lineage_graph) + + +def select_project_graph_impl( + *, graph: ProjectGraph, select: tuple[str, ...], exclude: tuple[str, ...] +) -> frozenset[CompiledObjectKey]: + """Resolve selectors on the graph's native handle.""" + + return resolve_graph_selection(graph=graph.native, select=select, exclude=exclude) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py b/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py index f6a639cc5d..e33a69dee9 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/relation_targets.py @@ -7,9 +7,9 @@ resolve_relation_location_qualified_name, ) from sqlbuild.compiler.compile.models import ( - CompiledProject, CompiledRelationLocation, ) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.references.main.render_source_relation import render_source_relation from sqlbuild.errors.contracts.exceptions import SharedInputError @@ -21,7 +21,7 @@ def build_python_relation_targets_impl( *, adapter: BaseAdapter, - project: CompiledProject, + project: CompiledProjectView, plan_output: PlanOutput, required_refs: frozenset[SqlResourceRef] | None = None, ) -> dict[SqlResourceRef, str]: @@ -77,7 +77,7 @@ def _planned_relation_refs(*, plan_output: PlanOutput) -> frozenset[SqlResourceR def _resolve_relation( *, adapter: BaseAdapter, - project: CompiledProject, + project: CompiledProjectView, plan_output: PlanOutput, ref: SqlResourceRef, ) -> str: diff --git a/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py b/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py index ef1b343622..a4d9953c01 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py @@ -4,74 +4,62 @@ from collections.abc import Callable from dataclasses import replace +from typing import cast +import sqlbuild._native as _native from sqlbuild.compiler.compile.models import ( - CompiledFunction, CompiledModel, - CompiledProject, CompiledRelationLocation, CompiledSeed, ) -from sqlbuild.compiler.compile.types import FunctionLanguage +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledSeedView, +) +from sqlbuild.compiler.compiled_project.main._with_target_defaults import with_target_defaults +from sqlbuild.compiler.compiled_project.main.project_with import project_with def apply_target_defaults( *, - project: CompiledProject, + project: CompiledProjectView, default_schema: str | None, default_database: str | None, render_qualified_name: Callable[..., str | None], python_functions_inherit_default_namespace: bool, -) -> CompiledProject: +) -> CompiledProjectView: """Apply adapter default schema/database to compiled project targets.""" - models: tuple[CompiledModel, ...] = tuple( - replace( - m, - destination=_resolve_target( - target=m.destination, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - ), + models: tuple[CompiledModelView, ...] = tuple( + _model_with_defaults( + model=model, + default_schema=default_schema, + default_database=default_database, + render_qualified_name=render_qualified_name, ) - for m in project.models + for model in project.models ) - seeds: tuple[CompiledSeed, ...] = tuple( - replace( - s, - destination=_resolve_target( - target=s.destination, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - ), + seeds: tuple[CompiledSeedView, ...] = tuple( + _seed_with_defaults( + seed=seed, + default_schema=default_schema, + default_database=default_database, + render_qualified_name=render_qualified_name, ) - for s in project.seeds + for seed in project.seeds ) - functions: tuple[CompiledFunction, ...] = tuple( - replace( - f, - destination=_resolve_function_target( - function=f, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - python_functions_inherit_default_namespace=( - python_functions_inherit_default_namespace - ), - ), - fingerprint_destination=_resolve_target( - target=f.fingerprint_destination, - default_schema=default_schema, - default_database=default_database, - render_qualified_name=render_qualified_name, - ), + functions: tuple[CompiledFunctionView, ...] = tuple( + cast(_native.NativeCompiledFunction, function).with_target_defaults( + (default_schema, default_database), + render_qualified_name, + python_functions_inherit_default_namespace, ) - for f in project.functions + for function in project.functions ) - return replace( - project, + return project_with( + project=project, effective_target_database=project.effective_target_database or default_database, effective_target_schema=project.effective_target_schema or default_schema, models=models, @@ -80,26 +68,48 @@ def apply_target_defaults( ) -def _resolve_function_target( +def _model_with_defaults( *, - function: CompiledFunction, + model: CompiledModelView, default_schema: str | None, default_database: str | None, render_qualified_name: Callable[..., str | None], - python_functions_inherit_default_namespace: bool, -) -> CompiledRelationLocation: - apply_defaults: bool = ( - function.language != FunctionLanguage.PYTHON or python_functions_inherit_default_namespace +) -> CompiledModelView: + if not isinstance(model, CompiledModel): + return with_target_defaults( + model=cast(_native.NativeCompiledModel, model), + defaults=(default_schema, default_database), + render_qualified_name=render_qualified_name, + ) + return replace( + model, + destination=_resolve_target( + target=model.destination, + default_schema=default_schema, + default_database=default_database, + render_qualified_name=render_qualified_name, + ), ) - resolved: CompiledRelationLocation = _resolve_target( - target=function.destination, - default_schema=default_schema if apply_defaults else None, - default_database=default_database if apply_defaults else None, - render_qualified_name=render_qualified_name, + + +def _seed_with_defaults( + *, + seed: CompiledSeedView, + default_schema: str | None, + default_database: str | None, + render_qualified_name: Callable[..., str | None], +) -> CompiledSeedView: + if isinstance(seed, _native.NativeCompiledSeed): + return seed.with_target_defaults((default_schema, default_database), render_qualified_name) + return replace( + cast(CompiledSeed, seed), + destination=_resolve_target( + target=seed.destination, + default_schema=default_schema, + default_database=default_database, + render_qualified_name=render_qualified_name, + ), ) - if not apply_defaults and resolved.qualified_name is None: - return replace(resolved, qualified_name=resolved.name) - return resolved def _resolve_target( diff --git a/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py b/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py index ec67be90c2..10cb92d949 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/target_validation.py @@ -11,9 +11,9 @@ from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config from sqlbuild.compiler.compile.main.expand_template_data import expand_template_data from sqlbuild.compiler.compile.models import ( - CompiledProject, CompiledRelationLocation, ) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.constants import DEFAULTS_SECTION, PROJECT_CONFIG_FILENAME from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.exceptions import PlannerInputError @@ -183,7 +183,7 @@ def _optional_string(value: object) -> str | None: return None if value is None else str(value) -def validate_project_targets(*, adapter_name: str, project: CompiledProject) -> None: +def validate_project_targets(*, adapter_name: str, project: CompiledProjectView) -> None: """Validate compiled model and seed locations for the effective adapter.""" if adapter_name not in { @@ -204,7 +204,7 @@ def validate_project_targets(*, adapter_name: str, project: CompiledProject) -> ) -def validate_managed_write_schemas(*, adapter: BaseAdapter, project: CompiledProject) -> None: +def validate_managed_write_schemas(*, adapter: BaseAdapter, project: CompiledProjectView) -> None: """Require an explicitly resolved physical schema for every managed warehouse write.""" if adapter.allows_implicit_managed_write_schema: diff --git a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py b/src/sqlbuild/compiler/pipeline/main/_compile_phase.py index 227657d846..983e2dae84 100644 --- a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py +++ b/src/sqlbuild/compiler/pipeline/main/_compile_phase.py @@ -8,8 +8,8 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config -from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline._helpers.analysis_selection import ( resolve_compile_analysis_selection, @@ -42,7 +42,7 @@ def compile_project_phase( on_progress("Compiling project...") compile_start: float = time.monotonic() try: - project: CompiledProject = build_compiled_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, selected_target=resolved_options.selected_target, diff --git a/src/sqlbuild/compiler/pipeline/main/compile.py b/src/sqlbuild/compiler/pipeline/main/compile.py index 6bb15d617f..7500686afe 100644 --- a/src/sqlbuild/compiler/pipeline/main/compile.py +++ b/src/sqlbuild/compiler/pipeline/main/compile.py @@ -12,16 +12,10 @@ from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, CompiledRelationLocation, ) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.graph.main._build_lineage_downstream_deps import ( - build_lineage_downstream_deps, -) -from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import ( - build_lineage_upstream_deps, -) from sqlbuild.compiler.pipeline._helpers.analysis_selection import ( apply_unit_test_selectors, resolve_configured_rule_selection, @@ -31,7 +25,7 @@ gather_deferred_relations, resolve_deferred_target_config, ) -from sqlbuild.compiler.pipeline._helpers.graph import build_static_all_keys +from sqlbuild.compiler.pipeline._helpers.graph import build_project_graph_impl from sqlbuild.compiler.pipeline._helpers.materializations import load_custom_materializations from sqlbuild.compiler.pipeline._helpers.python_plan_entries import ( build_python_run_plan_outputs, @@ -46,12 +40,6 @@ PythonRunPlanOutputs, ) from sqlbuild.compiler.planner.main.execution.execution import build_execution_plan -from sqlbuild.compiler.planner.main.selection._build_model_path_index import ( - build_model_path_index, -) -from sqlbuild.compiler.planner.main.selection._build_model_tag_index import ( - build_model_tag_index, -) from sqlbuild.compiler.planner.models import ( DeferralInputs, PlannerOverrides, @@ -96,9 +84,9 @@ def run_compile_pipeline( on_progress=on_progress, ) effective_config: dict[str, object] = compile_result.connection_config - project: CompiledProject = compile_result.project + project: CompiledProjectView = compile_result.project compile_seconds: float = compile_result.compile_seconds - graph: ProjectGraph = _build_project_graph(project=project) + graph: ProjectGraph = build_project_graph_impl(project) project_dir: Path | None = discovered_inputs.project_dir if project_dir is None: raise RulesError("compiler rules require a resolved project directory") @@ -177,7 +165,7 @@ def _build_result( discovered_inputs: DiscoveredProjectInputs, adapter: BaseAdapter, connection: Any, - project: CompiledProject, + project: CompiledProjectView, options: CompilePipelineOptions, on_progress: Callable[[str], None] | None = None, ) -> CompilePipelineResult: @@ -215,7 +203,7 @@ def _build_result( run_selection = resolve_python_sql_run_selection_from_inputs( select=select, exclude=exclude, - project_graph=_build_project_graph(project=project), + project_graph=build_project_graph_impl(project), discovered_inputs=discovered_inputs, ) selected_sql_keys = run_selection.sql_keys @@ -274,17 +262,3 @@ def _build_result( python_node_names=python_outputs.selected_python_node_names, python_plan_entries=python_outputs.python_plan_entries, ) - - -def _build_project_graph(*, project: CompiledProject) -> ProjectGraph: - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_upstream_deps(project) - ) - return ProjectGraph( - project=project, - upstream_deps=upstream_deps, - downstream_deps=build_lineage_downstream_deps(upstream_deps), - tag_index=build_model_tag_index(project), - path_index=build_model_path_index(project), - all_keys=build_static_all_keys(project), - ) diff --git a/src/sqlbuild/compiler/pipeline/main/compiled_project.py b/src/sqlbuild/compiler/pipeline/main/compiled_project.py index 748d8db834..133f17c259 100644 --- a/src/sqlbuild/compiler/pipeline/main/compiled_project.py +++ b/src/sqlbuild/compiler/pipeline/main/compiled_project.py @@ -2,7 +2,6 @@ from __future__ import annotations -from dataclasses import replace from functools import partial from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -12,9 +11,9 @@ from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileAnalysisSelection, - CompiledProject, - CompileProjectInputs, ) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.contracts.main.promotion_conflicts import promotion_conflict_diagnostics from sqlbuild.compiler.discovery.constants import ( LOCAL_CONFIG_FILENAME, @@ -55,7 +54,7 @@ def build_compiled_project( external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, resolved_connection: dict[str, object] | None = None, analysis_selection: CompileAnalysisSelection | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """Build one compiled project with adapter defaults and target validation applied.""" return compile_frontier( @@ -88,7 +87,7 @@ def _build_python_compiled_project( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, resolved_connection: dict[str, object] | None, analysis_selection: CompileAnalysisSelection | None, -) -> CompiledProject: +) -> CompiledProjectView: validate_named_target_schema_strategy( discovered_inputs=discovered_inputs, adapter=adapter, @@ -101,7 +100,7 @@ def _build_python_compiled_project( ) no_cache: bool = analysis_selection is not None and analysis_selection.no_cache with record_compile_timing("attachment_ms"): - compile_inputs: CompileProjectInputs = _frontier_compile_inputs( + compile_inputs: CompileInputsView = _frontier_compile_inputs( discovered_inputs=discovered_inputs, adapter=adapter, selected_target=selected_target, @@ -119,7 +118,7 @@ def _build_python_compiled_project( inference_profile=inference_profile, selection=analysis_selection, ) - project: CompiledProject = assemble_project( + project: CompiledProjectView = assemble_project( inputs=compile_inputs, inference_profile=inference_profile, skip_column_inference=skip_column_inference, @@ -144,8 +143,8 @@ def _build_python_compiled_project( ), project=project, ) - return replace( - project, + return project_with( + project=project, diagnostics=( *project.diagnostics, *promotion_conflict_diagnostics( @@ -172,7 +171,7 @@ def _frontier_compile_inputs( resolved_connection: dict[str, object] | None, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, no_cache: bool, -) -> CompileProjectInputs: +) -> CompileInputsView: return compile_frontier( until=CompilerStage.COMPILE_PROJECT_INPUTS, python_stage=partial( diff --git a/src/sqlbuild/compiler/pipeline/main/diff.py b/src/sqlbuild/compiler/pipeline/main/diff.py index ef29258261..dbdf62d710 100644 --- a/src/sqlbuild/compiler/pipeline/main/diff.py +++ b/src/sqlbuild/compiler/pipeline/main/diff.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline._helpers.diff import ( compile_project_for_diff_environment, @@ -24,10 +24,10 @@ def run_diff_pipeline( exclude: tuple[str, ...], cli_vars: dict[str, object] | None = None, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> tuple[CompiledProject, CompiledProject, tuple[str, ...]]: +) -> tuple[CompiledProjectView, CompiledProjectView, tuple[str, ...]]: """Compile both environments and resolve selected diffable model names.""" - left_project: CompiledProject = compile_project_for_diff_environment( + left_project: CompiledProjectView = compile_project_for_diff_environment( discovered_inputs=discovered_inputs, adapter=adapter, target_name=from_target, @@ -36,7 +36,7 @@ def run_diff_pipeline( cli_vars=cli_vars, external_sql_reference_resolver=external_sql_reference_resolver, ) - right_project: CompiledProject = compile_project_for_diff_environment( + right_project: CompiledProjectView = compile_project_for_diff_environment( discovered_inputs=discovered_inputs, adapter=adapter, target_name=to_target, diff --git a/src/sqlbuild/compiler/pipeline/main/project.py b/src/sqlbuild/compiler/pipeline/main/project.py index f44d69461f..50a337c728 100644 --- a/src/sqlbuild/compiler/pipeline/main/project.py +++ b/src/sqlbuild/compiler/pipeline/main/project.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver @@ -17,7 +17,7 @@ def compile_project( resolved_connection: dict[str, object] | None = None, no_sql_validation: bool = False, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """Compile discovered inputs into a target-defaulted project view.""" return build_compiled_project( diff --git a/src/sqlbuild/compiler/pipeline/main/project_graph.py b/src/sqlbuild/compiler/pipeline/main/project_graph.py index a455547c87..f90b53f4f4 100644 --- a/src/sqlbuild/compiler/pipeline/main/project_graph.py +++ b/src/sqlbuild/compiler/pipeline/main/project_graph.py @@ -2,37 +2,12 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject -from sqlbuild.compiler.graph.main._build_lineage_downstream_deps import ( - build_lineage_downstream_deps, -) -from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import ( - build_lineage_upstream_deps, -) -from sqlbuild.compiler.pipeline._helpers.graph import build_static_all_keys +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.pipeline._helpers.graph import build_project_graph_impl from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection._build_model_path_index import ( - build_model_path_index, -) -from sqlbuild.compiler.planner.main.selection._build_model_tag_index import ( - build_model_tag_index, -) -def build_project_graph_from_compiled_project(*, project: CompiledProject) -> ProjectGraph: +def build_project_graph_from_compiled_project(*, project: CompiledProjectView) -> ProjectGraph: """Build a static dependency graph for an already-compiled project.""" - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_upstream_deps(project) - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_downstream_deps(upstream_deps) - ) - return ProjectGraph( - project=project, - upstream_deps=upstream_deps, - downstream_deps=downstream_deps, - tag_index=build_model_tag_index(project), - path_index=build_model_path_index(project), - all_keys=build_static_all_keys(project), - ) + return build_project_graph_impl(project) diff --git a/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py b/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py new file mode 100644 index 0000000000..586feb3fa6 --- /dev/null +++ b/src/sqlbuild/compiler/pipeline/main/project_graph_selection.py @@ -0,0 +1,15 @@ +"""Selector resolution over a project graph.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.pipeline._helpers.graph import select_project_graph_impl +from sqlbuild.compiler.pipeline.models import ProjectGraph + + +def select_project_graph( + *, graph: ProjectGraph, select: tuple[str, ...], exclude: tuple[str, ...] +) -> frozenset[CompiledObjectKey]: + """Resolve `--select`/`--exclude` on the graph, adding the functions they need to build.""" + + return select_project_graph_impl(graph=graph, select=select, exclude=exclude) diff --git a/src/sqlbuild/compiler/pipeline/main/relation_targets.py b/src/sqlbuild/compiler/pipeline/main/relation_targets.py index cada81cd53..d5d2b1a38f 100644 --- a/src/sqlbuild/compiler/pipeline/main/relation_targets.py +++ b/src/sqlbuild/compiler/pipeline/main/relation_targets.py @@ -3,7 +3,7 @@ from __future__ import annotations from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers.relation_targets import ( build_python_relation_targets_impl, ) @@ -14,7 +14,7 @@ def build_python_relation_targets( *, adapter: BaseAdapter, - project: CompiledProject, + project: CompiledProjectView, plan_output: PlanOutput, required_refs: frozenset[SqlResourceRef] | None = None, ) -> dict[SqlResourceRef, str]: diff --git a/src/sqlbuild/compiler/pipeline/main/resolve_model_deferral_inputs.py b/src/sqlbuild/compiler/pipeline/main/resolve_model_deferral_inputs.py index c5828743dc..3f735d2777 100644 --- a/src/sqlbuild/compiler/pipeline/main/resolve_model_deferral_inputs.py +++ b/src/sqlbuild/compiler/pipeline/main/resolve_model_deferral_inputs.py @@ -6,7 +6,8 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline._helpers.deferred_locations import ( build_deferred_locations, @@ -19,7 +20,7 @@ def resolve_model_deferral_inputs( *, - project: CompiledProject, + project: CompiledProjectView, discovered_inputs: DiscoveredProjectInputs, adapter: BaseAdapter, connection: Any, diff --git a/src/sqlbuild/compiler/pipeline/main/selected_graph.py b/src/sqlbuild/compiler/pipeline/main/selected_graph.py index bc25d66ecc..883e0ce786 100644 --- a/src/sqlbuild/compiler/pipeline/main/selected_graph.py +++ b/src/sqlbuild/compiler/pipeline/main/selected_graph.py @@ -8,26 +8,13 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, - CompiledObjectKey, - CompiledProject, ) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.graph.main._build_lineage_downstream_deps import ( - build_lineage_downstream_deps, -) -from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import ( - build_lineage_upstream_deps, -) from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline._helpers.graph import build_static_all_keys +from sqlbuild.compiler.pipeline._helpers.graph import build_project_graph_impl from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.selection._build_model_path_index import ( - build_model_path_index, -) -from sqlbuild.compiler.planner.main.selection._build_model_tag_index import ( - build_model_tag_index, -) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver @@ -49,7 +36,7 @@ def build_project_graph_with_analysis_selection( if on_progress is not None: on_progress("Compiling project...") compile_start: float = time.monotonic() - project: CompiledProject = build_compiled_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, selected_target=selected_target, @@ -62,17 +49,4 @@ def build_project_graph_with_analysis_selection( ) if on_progress is not None: on_progress(f"Compiled project. ({time.monotonic() - compile_start:.2f}s)") - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_upstream_deps(project) - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_downstream_deps(upstream_deps) - ) - return ProjectGraph( - project=project, - upstream_deps=upstream_deps, - downstream_deps=downstream_deps, - tag_index=build_model_tag_index(project), - path_index=build_model_path_index(project), - all_keys=build_static_all_keys(project), - ) + return build_project_graph_impl(project) diff --git a/src/sqlbuild/compiler/pipeline/models.py b/src/sqlbuild/compiler/pipeline/models.py index 9a00d04fb6..6ab285d309 100644 --- a/src/sqlbuild/compiler/pipeline/models.py +++ b/src/sqlbuild/compiler/pipeline/models.py @@ -4,10 +4,15 @@ from collections.abc import Callable from dataclasses import dataclass, field +from functools import cached_property from typing import Any -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProviderUsage +from sqlbuild.compiler.graph.main._lineage_graph_views import lineage_graph_views +from sqlbuild.compiler.graph.models import LineageGraphViews from sqlbuild.compiler.planner.models import ( CloneSourcePlanEntry, CursorOverrides, @@ -29,14 +34,46 @@ @dataclass(frozen=True) class ProjectGraph: - """Static compiled project graph without warehouse state.""" + """Static compiled project graph held natively; dict views are built only when read.""" - project: CompiledProject - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - tag_index: dict[str, frozenset[CompiledObjectKey]] - path_index: dict[CompiledObjectKey, str] - all_keys: dict[str, CompiledObjectKey] + project: CompiledProjectView + native: _native.NativeProjectGraph = field(compare=False, repr=False) + + @cached_property + def lineage_views(self) -> LineageGraphViews: + """Lineage edges and selector indexes as dicts, converted from the native graph once.""" + + return lineage_graph_views(self.native) + + @property + def upstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: + """Lineage upstream edges, SQL tests stripped.""" + + return self.lineage_views.upstream_deps + + @property + def downstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: + """Lineage downstream edges sorted by `(resource type, name)`.""" + + return self.lineage_views.downstream_deps + + @property + def tag_index(self) -> dict[str, frozenset[CompiledObjectKey]]: + """Tag to tagged models, seeds and functions.""" + + return self.lineage_views.tag_index + + @property + def path_index(self) -> dict[CompiledObjectKey, str]: + """Model key to its folder below `models/`.""" + + return self.lineage_views.path_index + + @property + def all_keys(self) -> dict[str, CompiledObjectKey]: + """Selector name to key.""" + + return self.lineage_views.all_keys @dataclass(frozen=True) @@ -95,7 +132,7 @@ class PythonRunPlanOutputs: class CompilePipelineResult: """Complete output from the compile-and-plan pipeline.""" - project: CompiledProject + project: CompiledProjectView plan_output: PlanOutput custom_materializations: dict[str, Callable[..., Any]] = field(default_factory=dict) python_node_names: frozenset[str] = field(default_factory=frozenset) @@ -108,7 +145,7 @@ class CompilePipelineResult: class CompiledProjectPhaseResult: """Canonical project compilation output before command-specific planning.""" - project: CompiledProject + project: CompiledProjectView connection_config: dict[str, object] compile_seconds: float @@ -117,7 +154,7 @@ class CompiledProjectPhaseResult: class StaticCommandContext: """Shared compile, selection, and relation phases for focused commands.""" - project: CompiledProject + project: CompiledProjectView scope: PlannerScope relations: PlannerRelationsContext connection_config: dict[str, object] @@ -147,8 +184,8 @@ class ClonePipelineConnection: class ClonePipelineResult: """Prepared clone inputs for origin and destination target environments.""" - origin_project: CompiledProject - destination_project: CompiledProject + origin_project: CompiledProjectView + destination_project: CompiledProjectView clone_plan: PlanOutput destination_source_entries: tuple[CloneSourcePlanEntry, ...] = field(default_factory=tuple) destination_model_entries: tuple[ModelPlanEntry, ...] = field(default_factory=tuple) diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 9401c5338f..0ade165563 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -7,13 +7,15 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompiledObjectKey, - CompiledProject, InferredColumn, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.changes.metadata import ( changed_local_function_names, @@ -61,7 +63,7 @@ def detect_changes( *, - project: CompiledProject, + project: CompiledProjectView, scope: PlannerScope, snapshot: WarehouseSnapshot, full_refresh: bool, @@ -79,7 +81,7 @@ def detect_changes( ) function_changes: dict[str, FunctionChangeResult] = {} - function: CompiledFunction + function: CompiledFunctionView for function in project.functions: fingerprint_sql: str = build_compiled_function_fingerprint_sql(function) if function.key not in scope.selected_keys: @@ -104,7 +106,7 @@ def detect_changes( def detect_model_changes_in_scope( *, - project: CompiledProject, + project: CompiledProjectView, scope: PlannerScope, snapshot: WarehouseSnapshot, query_change_tracking: bool | None = None, @@ -130,7 +132,7 @@ def detect_model_changes_in_scope( for key in scope.execution_order: if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: continue - model: CompiledModel | None = scope.models_by_name.get(key.name) + model: CompiledModelView | None = scope.models_by_name.get(key.name) if model is None: continue model_changes[model.name] = detect_model_changes( @@ -149,7 +151,7 @@ def detect_model_changes_in_scope( def detect_model_changes( *, - model: CompiledModel, + model: CompiledModelView, snapshot: WarehouseSnapshot, sql_analysis_enabled: bool, query_change_tracking: bool, @@ -353,7 +355,7 @@ def detect_model_changes( ) -def _get_type_enforcement(model: CompiledModel) -> bool: +def _get_type_enforcement(model: CompiledModelView) -> bool: """Resolve whether type enforcement is active for a model.""" if model.schema_entry is not None and model.schema_entry.type_enforcement is not None: @@ -361,7 +363,7 @@ def _get_type_enforcement(model: CompiledModel) -> bool: return False -def _build_yml_columns(model: CompiledModel) -> tuple[ColumnInfo, ...]: +def _build_yml_columns(model: CompiledModelView) -> tuple[ColumnInfo, ...]: """Build expected columns from schema.yml declarations.""" if model.schema_entry is None: diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py index 8123e820a4..d0d7336477 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/recorded_renames.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.identity.hashing import model_definition_hash from sqlbuild.compiler.planner.classes.migration_state_inspection import ( @@ -55,7 +55,7 @@ def plan_reference_renames( def _has_changed_referencing_query(*, scope: PlannerScope, snapshot: WarehouseSnapshot) -> bool: - selected: tuple[CompiledModel, ...] = tuple( + selected: tuple[CompiledModelView, ...] = tuple( model for key in scope.selected_keys if (model := scope.models_by_name.get(key.name)) is not None @@ -63,7 +63,7 @@ def _has_changed_referencing_query(*, scope: PlannerScope, snapshot: WarehouseSn return any(_referencing_query_changed(model=model, snapshot=snapshot) for model in selected) -def _referencing_query_changed(*, model: CompiledModel, snapshot: WarehouseSnapshot) -> bool: +def _referencing_query_changed(*, model: CompiledModelView, snapshot: WarehouseSnapshot) -> bool: recorded: Fingerprint | None = snapshot.fingerprints.models.get(model.name) return ( recorded is not None diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py index 42410bfd89..c0b87e7151 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/seed_state.py @@ -6,14 +6,14 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledSeed +from sqlbuild.compiler.compile.types import CompiledSeedView from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME, NODE_TYPE_SEED from sqlbuild.compiler.fingerprints.main.read import read_latest_fingerprints from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet def read_seed_fingerprints( - *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeed, ...] + *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] ) -> dict[str, Fingerprint]: """Read only fingerprint state required to classify selected direct seeds.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py index b16b0a068d..a325947579 100644 --- a/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py +++ b/src/sqlbuild/compiler/planner/_helpers/command_planning/static.py @@ -3,7 +3,8 @@ from __future__ import annotations from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.identity.functions import ( build_compiled_function_fingerprint_sql, @@ -40,7 +41,7 @@ def resolve_static_command_scope_impl( *, - project: CompiledProject, + project: CompiledProjectView, selection: PlannerSelection, auto_load_sources: bool = False, ) -> PlannerScope: @@ -55,7 +56,7 @@ def resolve_static_command_scope_impl( def resolve_static_relation_context_impl( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, scope: PlannerScope, deferral: DeferralInputs | None = None, @@ -80,7 +81,7 @@ def resolve_static_relation_context_impl( def build_audit_command_plan_impl( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, scope: PlannerScope, relations: PlannerRelationsContext, @@ -108,7 +109,7 @@ def build_audit_command_plan_impl( def build_test_command_plan_impl( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, scope: PlannerScope, relations: PlannerRelationsContext, @@ -168,7 +169,7 @@ def build_test_command_plan_impl( def build_seed_command_plan_impl( *, - project: CompiledProject, + project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext, fingerprints: dict[str, Fingerprint] | None = None, @@ -212,7 +213,7 @@ def build_seed_command_plan_impl( def build_relation_command_plan_impl( - *, project: CompiledProject, scope: PlannerScope, relations: PlannerRelationsContext + *, project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext ) -> PlanOutput: """Project only static relation data for Python checks and scenarios.""" @@ -221,7 +222,7 @@ def build_relation_command_plan_impl( def _base_plan_output( *, - project: CompiledProject, + project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext, execution_order: tuple[CompiledObjectKey, ...] | None = None, diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py index 3d859ba3a7..139e393c53 100644 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py +++ b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py @@ -15,14 +15,16 @@ from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, - CompiledModel, - CompiledProject, CompileSqlReference, DynamicColumnContractProof, FixtureColumnInference, InferredColumn, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnTransformKind, @@ -47,7 +49,7 @@ def build_relation_fixture_completion( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, ordered_model_names: tuple[str, ...], fixture_groups: FixtureGroups, @@ -58,7 +60,7 @@ def build_relation_fixture_completion( context: RelationFixturePlanningContext = planning_context or build_relation_fixture_context( project=project ) - model_map: dict[str, CompiledModel] = context.models_by_name + model_map: dict[str, CompiledModelView] = context.models_by_name relations: dict[FixtureKey, FixtureRelationMetadata] = context.relations fixture_sql_by_key: dict[FixtureKey, str] inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] @@ -359,12 +361,12 @@ def _fixture_keys_requiring_analysis( def build_relation_fixture_context( *, - project: CompiledProject, - models_by_name: dict[str, CompiledModel] | None = None, + project: CompiledProjectView, + models_by_name: dict[str, CompiledModelView] | None = None, ) -> RelationFixturePlanningContext: """Build immutable project-wide fixture metadata once per planning invocation.""" - model_map: dict[str, CompiledModel] = ( + model_map: dict[str, CompiledModelView] = ( {model.name: model for model in project.models} if models_by_name is None else models_by_name @@ -397,7 +399,7 @@ def _authoritative_fixture_columns( def _required_fixture_columns( *, ordered_model_names: tuple[str, ...], - model_map: dict[str, CompiledModel], + model_map: dict[str, CompiledModelView], fixture_keys: frozenset[FixtureKey], relations: dict[FixtureKey, FixtureRelationMetadata], authoritative_columns: dict[FixtureKey, frozenset[str]], @@ -407,7 +409,7 @@ def _required_fixture_columns( star_fixture_keys: set[FixtureKey] = set() dialect: str | None = adapter.expression_inference_profile().sql_analysis_dialect for model_name in ordered_model_names: - model: CompiledModel | None = model_map.get(model_name) + model: CompiledModelView | None = model_map.get(model_name) if model is None: continue for upstream in _resolved_fixture_reads( @@ -498,7 +500,7 @@ def _cached_resolved_column_reads( def _star_fixture_keys( *, - model: CompiledModel, + model: CompiledModelView, fixture_keys: frozenset[FixtureKey], relations: dict[FixtureKey, FixtureRelationMetadata], adapter: BaseAdapter, @@ -591,7 +593,7 @@ def _replace_relation_markers_with_stubs(query_sql: str) -> str: def _fixture_relation_metadata( - *, project: CompiledProject, model_map: dict[str, CompiledModel] + *, project: CompiledProjectView, model_map: dict[str, CompiledModelView] ) -> dict[FixtureKey, FixtureRelationMetadata]: relations: dict[FixtureKey, FixtureRelationMetadata] = {} for model_name, model in model_map.items(): @@ -624,7 +626,7 @@ def _fixture_relation_metadata( return relations -def _model_fixture_metadata(*, model: CompiledModel) -> FixtureRelationMetadata: +def _model_fixture_metadata(*, model: CompiledModelView) -> FixtureRelationMetadata: columns_by_name: dict[str, FixtureColumnMetadata] = {} order: list[str] = [] for column in model.inferred_columns or (): @@ -665,11 +667,11 @@ def _expected_fixture_types( *, base_expected_types: dict[FixtureKey, dict[str, str]], ordered_model_names: tuple[str, ...], - model_map: dict[str, CompiledModel], + model_map: dict[str, CompiledModelView], ) -> Mapping[FixtureKey, dict[str, str]]: expected_types: ChainMap[FixtureKey, dict[str, str]] = ChainMap({}, base_expected_types) for model_name in ordered_model_names: - model: CompiledModel | None = model_map.get(model_name) + model: CompiledModelView | None = model_map.get(model_name) if model is None: continue model_key: FixtureKey = (CompiledResourceType.MODEL, model_name) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py b/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py index dc7b3bd1fb..8b347bc534 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/auto_load.py @@ -2,15 +2,15 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType def managed_source_upstream_keys( *, selected_keys: frozenset[CompiledObjectKey], upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - project: CompiledProject, + project: CompiledProjectView, ) -> frozenset[CompiledObjectKey]: """Return selected managed source keys reachable from the current selection.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/core.py b/src/sqlbuild/compiler/planner/_helpers/graph/core.py index c88875c2da..3757f956a4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/core.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/core.py @@ -6,18 +6,17 @@ from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, CompiledSqlTest, ) from sqlbuild.compiler.compile.types import ( + CompiledProjectView, CompiledResourceType, SqlTestMode, ) from sqlbuild.compiler.graph.main._attached_audit_gate_edges import attached_audit_gate_edges -from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import build_lineage_upstream_deps from sqlbuild.compiler.graph.main._hook_read_edges import hook_read_edges +from sqlbuild.compiler.graph.main._native_graph_key import native_graph_key from sqlbuild.compiler.graph.main.invert_edges import invert_edges -from sqlbuild.compiler.graph.main.path_nodes import path_nodes from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge from sqlbuild.compiler.planner.exceptions import PlannerInputError @@ -28,7 +27,7 @@ def _key_sort_key(key: CompiledObjectKey) -> tuple[str, str]: def build_execution_upstream_deps( - project: CompiledProject, + project: CompiledProjectView, ) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: """Return execution-order edges from lineage and woven SQL test nodes.""" @@ -64,7 +63,7 @@ def build_execution_upstream_deps( def build_execution_edge_origins( - project: CompiledProject, + project: CompiledProjectView, ) -> dict[tuple[CompiledObjectKey, CompiledObjectKey], str]: """Return human-readable origins for execution edges injected beyond pure lineage.""" @@ -75,12 +74,9 @@ def build_execution_edge_origins( origins[(target_key, test.key)] = ( f"SQL test '{test.name}' runs before '{target_key.name}'" ) - lineage_upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_upstream_deps(project) - ) edge: AttachedAuditGateEdge for edge in attached_audit_gate_edges(project=project): - if edge.read in lineage_upstream.get(edge.gated, ()): + if _is_lineage_edge(project=project, gated=edge.gated, read=edge.read): continue origins.setdefault( (edge.gated, edge.read), @@ -88,7 +84,7 @@ def build_execution_edge_origins( ) hook_edge: HookReadEdge for hook_edge in hook_read_edges(project=project): - if hook_edge.read in lineage_upstream.get(hook_edge.gated, ()): + if _is_lineage_edge(project=project, gated=hook_edge.gated, read=hook_edge.read): continue origins.setdefault( (hook_edge.gated, hook_edge.read), @@ -97,6 +93,12 @@ def build_execution_edge_origins( return origins +def _is_lineage_edge( + *, project: CompiledProjectView, gated: CompiledObjectKey, read: CompiledObjectKey +) -> bool: + return native_graph_key(read) in project.lineage_graph.upstream(native_graph_key(gated)) + + def _function_scope_deps_for_test(*, test: CompiledSqlTest) -> tuple[CompiledObjectKey, ...]: return tuple( scope_dep @@ -238,24 +240,3 @@ def expand_downstream( """Return all transitive downstream keys reachable from the given key.""" return transitive_closure(start=key, edges=downstream) - - -def find_path_keys( - *, - start: CompiledObjectKey, - end: CompiledObjectKey, - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[CompiledObjectKey]: - """Return all keys on directed paths from start to end.""" - - on_path: frozenset[CompiledObjectKey] | None = path_nodes( - start=start, - end=end, - downstream=downstream, - ) - if on_path is None: - raise PlannerInputError( - f"'{end.resource_type}:{end.name}' is not downstream of " - f"'{start.resource_type}:{start.name}'" - ) - return on_path diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py b/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py index 29b1e2ccc2..086cd9a8fe 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/loader_dag.py @@ -2,8 +2,8 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts from sqlbuild.spec.contracts.models import LoaderDestinationParts, SourceEntry @@ -11,7 +11,7 @@ def expand_selected_loader_dependencies( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey], upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], executable_dependency_source_keys: frozenset[CompiledObjectKey] = frozenset(), @@ -80,7 +80,7 @@ def expand_selected_loader_dependencies( def build_intermediate_source_map( - *, project: CompiledProject, selected_keys: frozenset[CompiledObjectKey] + *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] ) -> dict[str, SourceEntry]: """Return synthetic source entries for selected intermediate loader nodes.""" @@ -108,7 +108,7 @@ def build_intermediate_source_map( def build_upstream_intermediate_source_map( - *, project: CompiledProject, selected_keys: frozenset[CompiledObjectKey] + *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] ) -> dict[str, SourceEntry]: """Return synthetic source entries for upstream intermediate loaders of selected sources.""" @@ -189,7 +189,7 @@ def upstream_loader_dependency_names( def loader_to_source_entry( - *, project: CompiledProject, loader: DiscoveredLoaderFunction + *, project: CompiledProjectView, loader: DiscoveredLoaderFunction ) -> SourceEntry: destination: LoaderDestinationParts = loader_destination_parts( destination=( diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py index 194c20e6b8..04af510c1f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/scope.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/scope.py @@ -2,21 +2,10 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledSeed, - CompiledSource, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.graph.main._build_lineage_downstream_deps import ( - build_lineage_downstream_deps, -) -from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import ( - build_lineage_upstream_deps, -) +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.graph.main._lineage_graph_names import lineage_graph_names +from sqlbuild.compiler.graph.main._native_project_graph import build_native_project_graph from sqlbuild.compiler.planner._helpers.graph.auto_load import managed_source_upstream_keys from sqlbuild.compiler.planner._helpers.graph.core import ( build_downstream_deps, @@ -25,13 +14,10 @@ topologically_order_keys, ) from sqlbuild.compiler.planner._helpers.graph.loader_dag import expand_selected_loader_dependencies -from sqlbuild.compiler.planner._helpers.graph.selector_indexes import ( - build_model_path_index_impl as build_model_path_index, +from sqlbuild.compiler.planner._helpers.graph.selectors import ( + parse_selector, + resolve_graph_selectors, ) -from sqlbuild.compiler.planner._helpers.graph.selector_indexes import ( - build_model_tag_index_impl as build_model_tag_index, -) -from sqlbuild.compiler.planner._helpers.graph.selectors import parse_selector, resolve_selectors from sqlbuild.compiler.planner.models import ( ParsedSelector, PathSelector, @@ -43,7 +29,7 @@ def build_planner_scope( *, - project: CompiledProject, + project: CompiledProjectView, select: tuple[str, ...], exclude: tuple[str, ...], auto_load_sources: bool, @@ -57,15 +43,12 @@ def build_planner_scope( downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( upstream_deps ) - all_keys: dict[str, CompiledObjectKey] = _build_all_keys(project) + all_keys: dict[str, CompiledObjectKey] = lineage_graph_names(project.lineage_graph) resolved_selected_keys: frozenset[CompiledObjectKey] = ( selected_keys if selected_keys is not None - else _resolve_selection_on_lineage_graph( - project=project, - select=select, - exclude=exclude, - all_keys=all_keys, + else resolve_graph_selectors( + graph=build_native_project_graph(project), select=select, exclude=exclude ) ) executable_dependency_source_keys: frozenset[CompiledObjectKey] = ( @@ -112,46 +95,6 @@ def build_planner_scope( ) -def _resolve_selection_on_lineage_graph( - *, - project: CompiledProject, - select: tuple[str, ...], - exclude: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], -) -> frozenset[CompiledObjectKey]: - """Resolve user selection against the LINEAGE graph; execution edges never affect selection.""" - - lineage_upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_upstream_deps(project) - ) - return resolve_selectors( - select=select, - exclude=exclude, - all_keys=all_keys, - upstream=lineage_upstream, - downstream=build_lineage_downstream_deps(lineage_upstream), - tag_index=build_model_tag_index(project), - path_index=build_model_path_index(project), - ) - - -def _build_all_keys(project: CompiledProject) -> dict[str, CompiledObjectKey]: - keys: dict[str, CompiledObjectKey] = {} - model: CompiledModel - for model in project.models: - keys[model.name] = model.key - source: CompiledSource - for source in project.sources: - keys[source.name] = source.key - seed: CompiledSeed - for seed in project.seeds: - keys[seed.name] = seed.key - function: CompiledFunction - for function in project.functions: - keys[function.name] = function.key - return keys - - def _upstream_source_selector_keys( *, select: tuple[str, ...], diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/selector_indexes.py b/src/sqlbuild/compiler/planner/_helpers/graph/selector_indexes.py deleted file mode 100644 index 5fc100441e..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/graph/selector_indexes.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Project resource selector index implementations.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledSeed, -) -from sqlbuild.compiler.planner.constants import MODEL_SELECTOR_ROOT, MODEL_SELECTOR_ROOT_PREFIX - - -def build_model_tag_index_impl( - project: CompiledProject, -) -> dict[str, frozenset[CompiledObjectKey]]: - """Build a tag-to-keys lookup from compiled models, seeds, and functions.""" - - index: dict[str, set[CompiledObjectKey]] = {} - model: CompiledModel - for model in project.models: - tag: str - for tag in _as_string_list(model.config.values.get("tags")): - index.setdefault(tag, set()).add(model.key) - seed: CompiledSeed - for seed in project.seeds: - for tag in seed.schema_entry.tags: - index.setdefault(tag, set()).add(seed.key) - function: CompiledFunction - for function in project.functions: - for tag in function.tags: - index.setdefault(tag, set()).add(function.key) - return {tag: frozenset(keys) for tag, keys in index.items()} - - -def build_model_path_index_impl(project: CompiledProject) -> dict[CompiledObjectKey, str]: - """Build a key-to-folder lookup from compiled model relative paths.""" - - index: dict[CompiledObjectKey, str] = {} - model: CompiledModel - for model in project.models: - parent: str = str(model.relative_path.parent).replace("\\", "/") - index[model.key] = _strip_models_prefix(parent) - return index - - -def _strip_models_prefix(path: str) -> str: - if path.startswith(MODEL_SELECTOR_ROOT_PREFIX): - return path[len(MODEL_SELECTOR_ROOT_PREFIX) :] - if path == MODEL_SELECTOR_ROOT: - return "" - return path - - -def _as_string_list(value: object) -> list[str]: - if isinstance(value, list | tuple): - return [str(item) for item in value] - return [] diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py b/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py index d4c0cbbee0..83824206d4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/selectors.py @@ -1,111 +1,117 @@ -"""Selector parsing and scope resolution for planner graph selection.""" +"""Planner selector boundary over the native selector grammar and graph resolution.""" from __future__ import annotations -from collections.abc import Iterable -from difflib import get_close_matches -from fnmatch import fnmatchcase +from collections.abc import Callable, Iterable, Iterator, Mapping +import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner._helpers.graph.core import ( - expand_downstream, - expand_upstream, - find_path_keys, -) +from sqlbuild.compiler.graph.main._compiled_graph_keys import compiled_graph_keys +from sqlbuild.compiler.graph.main._native_graph_from_views import native_graph_from_views +from sqlbuild.compiler.graph.main._native_graph_keys import native_graph_keys from sqlbuild.compiler.planner.constants import ( - EMPTY_SELECTOR_PATH, - MODEL_SELECTOR_ROOT, - MODEL_SELECTOR_ROOT_PREFIX, - PATH_SELECTOR_EXPLICIT_ROOT_ERROR, - PATH_SELECTOR_SEPARATOR, - SELECTOR_KIND_SEPARATOR, - SELECTOR_MISSING_NAME_ERROR_FRAGMENT, - SELECTOR_PATH_SEPARATOR, - SELECTOR_SUGGESTION_CUTOFF, - SELECTOR_SUGGESTION_LIMIT, - SQL_FILE_SELECTOR_SUFFIX, UNIT_TEST_SELECTOR_ERROR_CODE, UNIT_TEST_SELECTOR_ONLY_TEST_AND_BUILD, ) from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion -from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector, SelectorExpansion +from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector from sqlbuild.compiler.planner.types import SelectorKind -from sqlbuild.errors.contracts.exceptions import SharedInputError - -_SELECTOR_KIND_BY_PREFIX: dict[str, SelectorKind] = { - SelectorKind.SEED: SelectorKind.SEED, - SelectorKind.SOURCE: SelectorKind.SOURCE, - SelectorKind.TASK: SelectorKind.TASK, - SelectorKind.ASSET: SelectorKind.ASSET, - SelectorKind.LOADER: SelectorKind.LOADER, - SelectorKind.CHECK: SelectorKind.CHECK, - SelectorKind.TAG: SelectorKind.TAG, - SelectorKind.PATH: SelectorKind.PATH, - SelectorKind.TEST: SelectorKind.TEST, -} - -_RESOURCE_TYPE_BY_SELECTOR_KIND: dict[SelectorKind, CompiledResourceType] = { - SelectorKind.SEED: CompiledResourceType.SEED, - SelectorKind.SOURCE: CompiledResourceType.SOURCE, -} + +_KIND_SELECTOR: str = "kind" +_SURROGATE_START: int = 0xD800 +_SURROGATE_END: int = 0xDFFF +_PRIVATE_USE_START: int = 0xF0000 +_PRIVATE_USE_END: int = 0xFFFFE + +type _Edges = Mapping[CompiledObjectKey, tuple[CompiledObjectKey, ...]] +type _Failure = tuple[str, str, str | None] +type _Outcome = tuple[list[tuple[str, str]] | None, _Failure | None] + + +class _SurrogateCarrier: + """Carry lone surrogates through the UTF-8 grammar as unused private-use characters.""" + + def __init__(self, *, texts: Iterable[str], reserved: Callable[[], Iterable[str]]) -> None: + joined: str = "".join(texts) + surrogates: list[str] = sorted( + {character for character in joined if _is_surrogate(character)} + ) + self._sent: dict[str, str] = {} + self._back: dict[str, str] = {} + if not surrogates: + return + taken: set[str] = set(joined).union(*map(set, reserved())) + candidates: Iterator[str] = ( + chr(point) for point in range(_PRIVATE_USE_START, _PRIVATE_USE_END) + ) + for surrogate, carrier in zip( + surrogates, + (character for character in candidates if character not in taken), + strict=False, + ): + self._sent[surrogate] = carrier + self._back[carrier] = surrogate + + def send(self, text: str) -> str: + return text.translate(str.maketrans(self._sent)) if self._sent else text + + def back(self, text: str | None) -> str | None: + if text is None or not self._back: + return text + return text.translate(str.maketrans(self._back)) + + def failure(self, failure: _Failure | None) -> _Failure | None: + if failure is None: + return None + code, message, help_text = failure + return (code, self.back(message) or "", self.back(help_text)) + + +def _is_surrogate(character: str) -> bool: + return _SURROGATE_START <= ord(character) <= _SURROGATE_END + + +def _graph_names(graph: _native.NativeProjectGraph) -> Callable[[], Iterable[str]]: + return lambda: (name for name, _ in graph.names()) def parse_selector(raw: str) -> ParsedSelector | PathSelector: """Parse one raw selector token into a structured form.""" - stripped: str = raw.strip() - try: - expansion: SelectorExpansion = split_selector_expansion(raw) - except SharedInputError as error: - code: str = "S001" if not stripped else "S002" - if SELECTOR_MISSING_NAME_ERROR_FRAGMENT in str(error): - code = "S004" - raise PlannerInputError(str(error), code=code) from None - - upstream: bool = expansion.upstream - downstream: bool = expansion.downstream - core: str = expansion.core - - if PATH_SELECTOR_SEPARATOR in core: - parts: list[str] = core.split(PATH_SELECTOR_SEPARATOR, 1) - start_name: str = parts[0].strip() - end_name: str = parts[1].strip() - if not start_name or not end_name: - raise PlannerInputError( - f"path selector '{stripped}' requires names on both sides of '~'", - code="S003", - ) - return PathSelector( - start_name=start_name, - end_name=end_name, + carrier: _SurrogateCarrier = _SurrogateCarrier(texts=(raw,), reserved=tuple) + parsed, failure = _native.parse_project_selector(carrier.send(raw)) + if parsed is None: + raise _selector_error(carrier.failure(failure)) + shape, first, second, upstream, downstream = parsed + if shape == _KIND_SELECTOR: + return ParsedSelector( + kind=SelectorKind(first), + value=carrier.back(second) or "", upstream=upstream, downstream=downstream, ) + return PathSelector( + start_name=carrier.back(first) or "", + end_name=carrier.back(second) or "", + upstream=upstream, + downstream=downstream, + ) - name: str = core - if SELECTOR_KIND_SEPARATOR in name: - prefix: str - value: str - prefix, value = name.split(SELECTOR_KIND_SEPARATOR, 1) - kind: SelectorKind | None = _SELECTOR_KIND_BY_PREFIX.get(prefix) - if kind is None: - raise PlannerInputError( - f"unknown selector type '{prefix}' in '{stripped}'", code="S005" - ) - if not value: - raise PlannerInputError(f"selector '{stripped}' has empty value after ':'", code="S006") - return ParsedSelector(kind=kind, value=value, upstream=upstream, downstream=downstream) - - if SELECTOR_PATH_SEPARATOR in name: - folder_value: str = name.strip(SELECTOR_PATH_SEPARATOR) - return ParsedSelector( - kind=SelectorKind.PATH, value=folder_value, upstream=upstream, downstream=downstream - ) - return ParsedSelector( - kind=SelectorKind.NAME, value=name, upstream=upstream, downstream=downstream +def resolve_graph_selectors( + *, + graph: _native.NativeProjectGraph, + select: tuple[str, ...], + exclude: tuple[str, ...], +) -> frozenset[CompiledObjectKey]: + """Resolve select/exclude strings against a native graph, adding required functions.""" + + carrier: _SurrogateCarrier = _SurrogateCarrier( + texts=(*select, *exclude), reserved=_graph_names(graph) + ) + return _selected( + outcome=graph.resolve(list(map(carrier.send, select)), list(map(carrier.send, exclude))), + carrier=carrier, ) @@ -113,337 +119,94 @@ def resolve_selectors( *, select: tuple[str, ...], exclude: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]] | None = None, - path_index: dict[CompiledObjectKey, str] | None = None, + all_keys: Mapping[str, CompiledObjectKey], + upstream: _Edges, + downstream: _Edges, + tag_index: Mapping[str, Iterable[CompiledObjectKey]] | None = None, + path_index: Mapping[CompiledObjectKey, str] | None = None, ) -> frozenset[CompiledObjectKey]: """Resolve raw select/exclude strings into a final set of object keys.""" - effective_tag_index: dict[str, frozenset[CompiledObjectKey]] = tag_index or {} - effective_path_index: dict[CompiledObjectKey, str] = path_index or {} - - if not select and not exclude: - return frozenset(all_keys.values()) - - selected: frozenset[CompiledObjectKey] = ( - resolve_selector_tokens( - selectors=select, - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=effective_tag_index, - path_index=effective_path_index, - ) - if select - else frozenset(all_keys.values()) - ) - excluded: frozenset[CompiledObjectKey] = resolve_selector_tokens( - selectors=exclude, + graph: _native.NativeProjectGraph = native_graph_from_views( all_keys=all_keys, upstream=upstream, downstream=downstream, - tag_index=effective_tag_index, - path_index=effective_path_index, - ) - - scoped: frozenset[CompiledObjectKey] = selected - excluded - return expand_required_build_resources( - selected_keys=scoped, - upstream=upstream, - downstream=downstream, - include_upstream_functions=True, - include_upstream_seeds=False, - include_downstream_functions=False, + tag_index=tag_index or {}, + path_index=path_index or {}, ) + return resolve_graph_selectors(graph=graph, select=select, exclude=exclude) def resolve_selector_tokens( *, selectors: tuple[str, ...], - all_keys: dict[str, CompiledObjectKey], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], + all_keys: Mapping[str, CompiledObjectKey], + upstream: _Edges, + downstream: _Edges, + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], ) -> frozenset[CompiledObjectKey]: """Union the keys matched by whitespace-separated selector tokens, without build expansion.""" - resolved: set[CompiledObjectKey] = set() - raw_selector: str - for raw_selector in selectors: - token: str - for token in raw_selector.split(): - resolved.update( - _resolve_token( - token=token, - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index, - path_index=path_index, - ) - ) - return frozenset(resolved) + graph: _native.NativeProjectGraph = native_graph_from_views( + all_keys=all_keys, + upstream=upstream, + downstream=downstream, + tag_index=tag_index, + path_index=path_index, + ) + carrier: _SurrogateCarrier = _SurrogateCarrier(texts=selectors, reserved=_graph_names(graph)) + return _selected(outcome=graph.tokens(list(map(carrier.send, selectors))), carrier=carrier) def match_selector_keys( *, parsed: ParsedSelector, - all_keys: dict[str, CompiledObjectKey], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], + all_keys: Mapping[str, CompiledObjectKey], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], ) -> frozenset[CompiledObjectKey]: """Return the keys one parsed selector matches before `+` graph expansion.""" - if parsed.kind == SelectorKind.TAG: - tagged_keys: frozenset[CompiledObjectKey] = tag_index.get(parsed.value, frozenset()) - if not tagged_keys: - raise PlannerInputError(f"no models found with tag '{parsed.value}'", code="S008") - return tagged_keys - - if parsed.kind == SelectorKind.PATH: - return _match_path(value=parsed.value, path_index=path_index) - - if parsed.kind == SelectorKind.TEST: - raise unit_test_selector_rejection( - selector=f"{SelectorKind.TEST}{SELECTOR_KIND_SEPARATOR}{parsed.value}" - ) - - keys: frozenset[CompiledObjectKey] = _lookup_keys(parsed=parsed, all_keys=all_keys) - if not keys: - label: str = "pattern" if _is_name_pattern(parsed.value) else "name" - raise PlannerInputError( - f"unknown selector {label} '{parsed.value}'", - code="S007", - help=selector_name_help(value=parsed.value, candidates=all_keys), - ) - return keys + graph: _native.NativeProjectGraph = native_graph_from_views( + all_keys=all_keys, upstream={}, downstream={}, tag_index=tag_index, path_index=path_index + ) + carrier: _SurrogateCarrier = _SurrogateCarrier( + texts=(parsed.value,), reserved=_graph_names(graph) + ) + return _selected( + outcome=graph.matched(str(parsed.kind), carrier.send(parsed.value)), carrier=carrier + ) def expand_required_build_resources( *, selected_keys: frozenset[CompiledObjectKey], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], + upstream: _Edges, + downstream: _Edges, include_upstream_functions: bool = True, include_upstream_seeds: bool = False, include_downstream_functions: bool = False, ) -> frozenset[CompiledObjectKey]: """Add non-model resources needed to build a coherent selected model scope.""" - expanded: set[CompiledObjectKey] = set(selected_keys) - key: CompiledObjectKey - for key in tuple(selected_keys): - upstream_key: CompiledObjectKey - for upstream_key in expand_upstream(key=key, upstream=upstream): - if include_upstream_functions and upstream_key.resource_type in { - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - }: - expanded.add(upstream_key) - if include_upstream_seeds and upstream_key.resource_type == CompiledResourceType.SEED: - expanded.add(upstream_key) - if not include_downstream_functions: - return frozenset(expanded) - selected_model_keys: frozenset[CompiledObjectKey] = frozenset( - key for key in selected_keys if key.resource_type == CompiledResourceType.MODEL + graph: _native.NativeProjectGraph = native_graph_from_views( + all_keys={}, upstream=upstream, downstream=downstream, tag_index={}, path_index={} ) - for key in selected_model_keys: - downstream_key: CompiledObjectKey - for downstream_key in downstream.get(key, ()): - if downstream_key.resource_type in { - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - }: - expanded.add(downstream_key) - return frozenset(expanded) - - -def _resolve_token( - *, - token: str, - all_keys: dict[str, CompiledObjectKey], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Resolve one selector token, handling comma intersection.""" - - parts: list[str] = token.split(",") - if len(parts) == 1: - return _resolve_single( - raw=parts[0], - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index, - path_index=path_index, - ) - - sets: list[frozenset[CompiledObjectKey]] = [ - _resolve_single( - raw=part, - all_keys=all_keys, - upstream=upstream, - downstream=downstream, - tag_index=tag_index, - path_index=path_index, - ) - for part in parts - ] - result: frozenset[CompiledObjectKey] = sets[0] - subsequent: frozenset[CompiledObjectKey] - for subsequent in sets[1:]: - result = result & subsequent - return result - - -def _resolve_single( - *, - raw: str, - all_keys: dict[str, CompiledObjectKey], - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], - tag_index: dict[str, frozenset[CompiledObjectKey]], - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Resolve one atomic selector (no commas).""" - - parsed: ParsedSelector | PathSelector = parse_selector(raw) - - if isinstance(parsed, PathSelector): - start_name: str = parsed.start_name - end_name: str = parsed.end_name - start_key: CompiledObjectKey | None = all_keys.get(start_name) - end_key: CompiledObjectKey | None = all_keys.get(end_name) - if start_key is None: - raise PlannerInputError(f"unknown selector name '{start_name}'", code="S007") - if end_key is None: - raise PlannerInputError(f"unknown selector name '{end_name}'", code="S007") - result: set[CompiledObjectKey] = set( - find_path_keys(start=start_key, end=end_key, downstream=downstream) + return compiled_graph_keys( + graph.build_resources( + native_graph_keys(selected_keys), + (include_upstream_functions, include_upstream_seeds, include_downstream_functions), ) - if parsed.upstream: - result.update(expand_upstream(key=start_key, upstream=upstream)) - if parsed.downstream: - result.update(expand_downstream(key=end_key, downstream=downstream)) - return frozenset(result) - - keys: frozenset[CompiledObjectKey] = match_selector_keys( - parsed=parsed, - all_keys=all_keys, - tag_index=tag_index, - path_index=path_index, - ) - result: set[CompiledObjectKey] = set(keys) - key: CompiledObjectKey - if parsed.upstream: - for key in keys: - result.update(expand_upstream(key=key, upstream=upstream)) - if parsed.downstream: - for key in keys: - result.update(expand_downstream(key=key, downstream=downstream)) - return frozenset(result) - - -def _match_path( - *, - value: str, - path_index: dict[CompiledObjectKey, str], -) -> frozenset[CompiledObjectKey]: - """Match a `models/`-rooted path selector against model folders.""" - - folder: str = _normalize_path_selector_value(value) - selector_folder: str = _model_path_candidate(folder) - matched_keys: frozenset[CompiledObjectKey] = frozenset( - key - for key, indexed_folder in path_index.items() - if _path_matches(indexed_folder=indexed_folder, selector_folder=selector_folder) - ) - if not matched_keys: - raise PlannerInputError( - f"no models found under path '{folder}'." - + ( - " Path selectors match folders; select a single model by its name." - if folder.endswith(SQL_FILE_SELECTOR_SUFFIX) - else "" - ), - code="S009", - ) - return matched_keys - - -def _normalize_path_selector_value(value: str) -> str: - return value.replace("\\", "/").strip("/") - - -def _model_path_candidate(folder: str) -> str: - if folder == MODEL_SELECTOR_ROOT: - return EMPTY_SELECTOR_PATH - if folder.startswith(MODEL_SELECTOR_ROOT_PREFIX): - return folder[len(MODEL_SELECTOR_ROOT_PREFIX) :] - raise PlannerInputError( - PATH_SELECTOR_EXPLICIT_ROOT_ERROR, - code="S012", ) -def _path_matches(*, indexed_folder: str, selector_folder: str) -> bool: - if selector_folder == EMPTY_SELECTOR_PATH: - return True - return indexed_folder == selector_folder or indexed_folder.startswith(f"{selector_folder}/") - - -def _lookup_keys( - *, - parsed: ParsedSelector, - all_keys: dict[str, CompiledObjectKey], -) -> frozenset[CompiledObjectKey]: - """Look up object keys for an exact or glob-pattern name selector.""" - - if parsed.kind == SelectorKind.NAME: - if not _is_name_pattern(parsed.value): - candidate: CompiledObjectKey | None = all_keys.get(parsed.value) - return frozenset() if candidate is None else frozenset((candidate,)) - return frozenset(key for name, key in all_keys.items() if fnmatchcase(name, parsed.value)) - - resource_type: CompiledResourceType | None = _RESOURCE_TYPE_BY_SELECTOR_KIND.get(parsed.kind) - if resource_type is None: - raise PlannerInputError( - f"selector type '{parsed.kind}' does not map to a resource type yet", - code="S010", - ) - - if not _is_name_pattern(parsed.value): - candidate = all_keys.get(parsed.value) - if candidate is not None and candidate.resource_type == resource_type: - return frozenset((candidate,)) - return frozenset() - return frozenset( - key - for name, key in all_keys.items() - if key.resource_type == resource_type and fnmatchcase(name, parsed.value) - ) - - -def _is_name_pattern(value: str) -> bool: - return any(character in value for character in "*?[") - - def selector_name_help(*, value: str, candidates: Iterable[str]) -> str | None: """Suggest the nearest selectable names for an unknown selector name.""" - if _is_name_pattern(value): - return None - matches: list[str] = get_close_matches( - value, sorted(candidates), n=SELECTOR_SUGGESTION_LIMIT, cutoff=SELECTOR_SUGGESTION_CUTOFF - ) - if not matches: - return None - return f"did you mean {', '.join(f"'{match}'" for match in matches)}?" + names: list[str] = list(candidates) + carrier: _SurrogateCarrier = _SurrogateCarrier(texts=(value,), reserved=lambda: names) + return carrier.back(_native.project_selector_name_help(carrier.send(value), names)) def unit_test_selector_rejection(*, selector: str) -> PlannerInputError: @@ -453,3 +216,17 @@ def unit_test_selector_rejection(*, selector: str) -> PlannerInputError: f"selector '{selector}' selects a unit test; {UNIT_TEST_SELECTOR_ONLY_TEST_AND_BUILD}", code=UNIT_TEST_SELECTOR_ERROR_CODE, ) + + +def _selected(*, outcome: _Outcome, carrier: _SurrogateCarrier) -> frozenset[CompiledObjectKey]: + keys, failure = outcome + if keys is None: + raise _selector_error(carrier.failure(failure)) + return compiled_graph_keys(keys) + + +def _selector_error(failure: _Failure | None) -> PlannerInputError: + if failure is None: + return PlannerInputError("selector resolution returned neither keys nor an error") + code, message, help_text = failure + return PlannerInputError(message, code=code, help=help_text) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py b/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py index 2e3218ca50..a4eb9e7ee8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/source_load_nodes.py @@ -2,8 +2,8 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( build_intermediate_source_map, @@ -16,7 +16,7 @@ def build_source_load_map( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey], upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] | None = None, ) -> dict[str, SourceEntry]: diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py index f67daea363..ebbb499166 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py @@ -2,8 +2,12 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledFunction, CompiledModel, CompiledSeed -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledResourceType, + CompiledSeedView, +) from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner._helpers.identity.hashing import ( build_model_local_identity_hash, @@ -33,8 +37,8 @@ def build_direct_model_version_identities( *, - functions: tuple[CompiledFunction, ...], - seeds: tuple[CompiledSeed, ...] = (), + functions: tuple[CompiledFunctionView, ...], + seeds: tuple[CompiledSeedView, ...] = (), scope: PlannerScope, source_version_hashes: dict[str, str] | None = None, hook_functions: tuple[DiscoveredHookFunction, ...] = (), @@ -51,7 +55,7 @@ def build_direct_model_version_identities( } seed_version_hashes: dict[str, str] = {} seed_metadata_jsons: dict[str, str] = {} - seed: CompiledSeed + seed: CompiledSeedView for seed in seeds: seed_hash: str seed_metadata_json: str @@ -61,7 +65,7 @@ def build_direct_model_version_identities( model_metadata_jsons: dict[str, str] = {} model_local_hashes: dict[str, str] = {} graph_nodes: dict[GraphNodeKey, GraphIdentityNode] = {} - function: CompiledFunction + function: CompiledFunctionView for function in functions: function_hash: str | None = function_local_hashes.get(function.name) if function_hash is None: @@ -113,7 +117,7 @@ def build_direct_model_version_identities( for key in scope.execution_order: if not hasattr(key, "resource_type") or key.resource_type != CompiledResourceType.MODEL: continue - model: CompiledModel | None = scope.models_by_name.get(key.name) + model: CompiledModelView | None = scope.models_by_name.get(key.name) if model is None: continue metadata_json: str = build_model_version_identity_metadata_json( diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py index 496645b184..ac2bb6b279 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/functions.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/functions.py @@ -3,16 +3,16 @@ from __future__ import annotations from sqlbuild.compiler.compile.models import ( - CompiledFunction, FunctionArgument, FunctionReturnColumn, ) +from sqlbuild.compiler.compile.types import CompiledFunctionView from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.identity.hashing import function_definition_hash from sqlbuild.compiler.planner.types import PlanReason -def build_compiled_function_fingerprint_sql(function: CompiledFunction) -> str: +def build_compiled_function_fingerprint_sql(function: CompiledFunctionView) -> str: """Build stable function definition text used for change fingerprints.""" return build_function_fingerprint_sql( @@ -29,7 +29,7 @@ def build_compiled_function_fingerprint_sql(function: CompiledFunction) -> str: def detect_function_change( *, - function: CompiledFunction, + function: CompiledFunctionView, fingerprint_sql: str, fingerprint: Fingerprint | None, query_change_tracking: bool, diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/honest.py b/src/sqlbuild/compiler/planner/_helpers/identity/honest.py index 368666166b..2e8cb7ee6f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/honest.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/honest.py @@ -4,8 +4,8 @@ from dataclasses import replace -from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.identity.hashing import ( compose_native_graph_identity, @@ -61,13 +61,13 @@ def with_honest_model_write_hashes( base_identity_hashes=base_hashes, compose_identity=compose_native_graph_identity, ) - models_by_name: dict[str, CompiledModel] = scope.models_by_name + models_by_name: dict[str, CompiledModelView] = scope.models_by_name model_changes: dict[str, ChangeDetectionResult] = dict(changes.models) key: CompiledObjectKey for key in scope.execution_order: if key.resource_type != CompiledResourceType.MODEL: continue - model: CompiledModel | None = models_by_name.get(key.name) + model: CompiledModelView | None = models_by_name.get(key.name) if model is None: continue if key not in scope.selected_keys: @@ -162,7 +162,7 @@ def _graph_identity_nodes( upstream_keys=(), local_hash=seed_hash, ) - model: CompiledModel + model: CompiledModelView for model in scope.models_by_name.values(): model_key: GraphNodeKey = graph_key_for_compiled_resource( resource_type=CompiledResourceType.MODEL, diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py b/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py index 119438a09d..00ec07931c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/model_metadata.py @@ -5,13 +5,13 @@ import hashlib import json -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import EnumDeclaration from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry -def contract_output_signature(*, model: CompiledModel) -> dict[str, object] | None: +def contract_output_signature(*, model: CompiledModelView) -> dict[str, object] | None: """Build required model output metadata that participates in execution identity.""" schema_entry: SchemaModelEntry | None = model.schema_entry @@ -41,7 +41,9 @@ def contract_output_signature(*, model: CompiledModel) -> dict[str, object] | No return signature -def _column_output_signature(*, model: CompiledModel, column: SchemaColumn) -> dict[str, object]: +def _column_output_signature( + *, model: CompiledModelView, column: SchemaColumn +) -> dict[str, object]: signature: dict[str, object] = { "name": column.name, "type": column.type, @@ -58,7 +60,7 @@ def _column_output_signature(*, model: CompiledModel, column: SchemaColumn) -> d return signature -def declared_columns_hash(*, model: CompiledModel) -> str: +def declared_columns_hash(*, model: CompiledModelView) -> str: """Hash the model's own declared columns, column families, and contract setting.""" schema_entry: SchemaModelEntry | None = model.schema_entry diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/seed.py b/src/sqlbuild/compiler/planner/_helpers/identity/seed.py index a1211436c0..e957ed8126 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/seed.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/seed.py @@ -8,19 +8,19 @@ from pathlib import Path from typing import Any -from sqlbuild.compiler.compile.models import CompiledSeed +from sqlbuild.compiler.compile.types import CompiledSeedView from sqlbuild.compiler.planner._helpers.identity.hashing import stable_version_identity_hash from sqlbuild.spec.contracts.models import SchemaColumn, SeedCsvSettings -def build_seed_identity(seed: CompiledSeed) -> tuple[str, str]: +def build_seed_identity(seed: CompiledSeedView) -> tuple[str, str]: """Return ``(identity_hash, metadata_json)`` for one compiled seed.""" metadata_json: str = build_seed_identity_metadata_json(seed) return stable_version_identity_hash(metadata_json), metadata_json -def build_seed_identity_metadata_json(seed: CompiledSeed) -> str: +def build_seed_identity_metadata_json(seed: CompiledSeedView) -> str: """Build deterministic seed identity metadata from content and load-affecting config.""" payload: dict[str, object] = { diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py index edf4edb742..7021e2161e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/columns.py @@ -6,8 +6,8 @@ from dataclasses import dataclass, replace from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.migrations.main._newest_column_event import ( newest_column_migration_event_mentioning, @@ -60,7 +60,7 @@ class _ModelColumns: """Inputs gathered for one model that may rename columns this run.""" - model: CompiledModel + model: CompiledModelView warehouse: dict[str, ColumnInfo] declared: dict[str, str] fingerprint: Fingerprint | None @@ -179,7 +179,7 @@ def _candidate( ) -> _ModelColumns | None: if key not in scope.selected_keys or key.resource_type != CompiledResourceType.MODEL: return None - model: CompiledModel | None = scope.models_by_name.get(key.name) + model: CompiledModelView | None = scope.models_by_name.get(key.name) if ( model is None or get_config_str(values=model.config.values, key="materialized") @@ -478,7 +478,7 @@ def _with_renamed_cursors( old_cursor_columns: dict[str, str] = {} entry: ColumnMigrationPlanEntry for entry in entries: - model: CompiledModel | None = scope.models_by_name.get(entry.model_name) + model: CompiledModelView | None = scope.models_by_name.get(entry.model_name) cursor: str | None = ( get_config_str(values=model.config.values, key="cursor") if model else None ) diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py index ccf1deae87..dc31b25b99 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/compatibility.py @@ -5,7 +5,7 @@ from sqlbuild.adapter.contract.models import ColumnInfo, NormalizedType, RelationInfo from sqlbuild.adapter.contract.types import TypeFamily from sqlbuild.adapter.type_system.main.normalize_type import normalize_type -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.migrations.types import MigrationCompatibility from sqlbuild.compiler.planner._helpers.changes.detect import detect_model_changes from sqlbuild.compiler.planner.constants import VIEW_RELATION_TYPE_MARKER @@ -35,7 +35,7 @@ def check_migration_compatibility( *, - model: CompiledModel, + model: CompiledModelView, origin_relation: RelationInfo, origin_columns: tuple[ColumnInfo, ...], sql_analysis_enabled: bool, @@ -82,7 +82,7 @@ def check_migration_compatibility( return MigrationCompatibilityResult(status=MigrationCompatibility.COMPATIBLE) -def _has_expected_columns(*, model: CompiledModel) -> bool: +def _has_expected_columns(*, model: CompiledModelView) -> bool: if model.schema_entry is not None and any( column.type is not None for column in model.schema_entry.columns ): @@ -91,7 +91,7 @@ def _has_expected_columns(*, model: CompiledModel) -> bool: def _incremental_blocking_findings( - *, model: CompiledModel, findings: tuple[SchemaFinding, ...], dialect: str | None + *, model: CompiledModelView, findings: tuple[SchemaFinding, ...], dialect: str | None ) -> tuple[str, ...]: raw_policy: str | None = get_config_str(values=model.config.values, key="on_schema_change") policy: OnSchemaChange = OnSchemaChange(raw_policy) if raw_policy else _DEFAULT_ON_SCHEMA_CHANGE @@ -110,7 +110,7 @@ def _incremental_blocking_findings( def _snapshot_blocking_findings( *, - model: CompiledModel, + model: CompiledModelView, findings: tuple[SchemaFinding, ...], dialect: str | None, snapshots_config: SnapshotsConfig, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index 6daf12837c..d560e13336 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -7,11 +7,10 @@ from typing import Any from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledObjectKey, CompiledRelationLocation, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning from sqlbuild.compiler.migrations.main.relation_for_location import ( @@ -68,7 +67,7 @@ def discover_model_migrations( ) -> ModelMigrationDiscovery: """Add unique one-to-one fingerprint matches to the explicit migration requests.""" - models: tuple[CompiledModel, ...] = _topological_models(runtime=runtime) + models: tuple[CompiledModelView, ...] = _topological_models(runtime=runtime) declared: frozenset[str] = frozenset(item.model_name for item in declarations) triggered: bool = bool( _new_model_names(models=models, scope=scope, snapshot=snapshot) - declared @@ -160,7 +159,10 @@ def _declared_renames( def _event_renames( - *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...], state: MigrationStateInspection + *, + runtime: PlannerRuntime, + models: tuple[CompiledModelView, ...], + state: MigrationStateInspection, ) -> dict[str, str]: """Map project models to the removed model their latest recorded move came from.""" @@ -183,7 +185,7 @@ def _event_renames( def _resumed_events( *, runtime: PlannerRuntime, - models: tuple[CompiledModel, ...], + models: tuple[CompiledModelView, ...], unbuilt: frozenset[str], state: MigrationStateInspection, ) -> dict[str, MigrationEvent]: @@ -191,7 +193,7 @@ def _resumed_events( target_name: str | None = runtime.project.effective_target_name resumed: dict[str, MigrationEvent] = {} - model: CompiledModel + model: CompiledModelView for model in models: if model.name not in unbuilt or not (_stores_history(model) or _hands_over_identity(model)): continue @@ -235,14 +237,14 @@ def _automatic_requests( *, runtime: PlannerRuntime, scope: PlannerScope, - models: tuple[CompiledModel, ...], + models: tuple[CompiledModelView, ...], matches: dict[str, Fingerprint], resumed: dict[str, MigrationEvent], ) -> tuple[ModelMigrationRequest, ...]: """Return automatic requests for selected destinations only.""" requests: list[ModelMigrationRequest] = [] - model: CompiledModel + model: CompiledModelView for model in models: if model.key not in scope.selected_keys: continue @@ -288,7 +290,7 @@ def _event_origin_location( def _ambiguity_warnings( - *, models: tuple[CompiledModel, ...], scope: PlannerScope, ambiguous: dict[str, str] + *, models: tuple[CompiledModelView, ...], scope: PlannerScope, ambiguous: dict[str, str] ) -> tuple[PlanWarning, ...]: selected: frozenset[str] = frozenset( model.name for model in models if model.key in scope.selected_keys @@ -308,9 +310,9 @@ def _ambiguity_warnings( ) -def _topological_models(*, runtime: PlannerRuntime) -> tuple[CompiledModel, ...]: - by_name: dict[str, CompiledModel] = {model.name: model for model in runtime.project.models} - ordered: list[CompiledModel] = [] +def _topological_models(*, runtime: PlannerRuntime) -> tuple[CompiledModelView, ...]: + by_name: dict[str, CompiledModelView] = {model.name: model for model in runtime.project.models} + ordered: list[CompiledModelView] = [] visited: set[str] = set() pending: list[tuple[str, bool]] = [(name, False) for name in sorted(by_name, reverse=True)] while pending: @@ -335,7 +337,7 @@ def _topological_models(*, runtime: PlannerRuntime) -> tuple[CompiledModel, ...] def _new_model_names( - *, models: tuple[CompiledModel, ...], scope: PlannerScope, snapshot: WarehouseSnapshot + *, models: tuple[CompiledModelView, ...], scope: PlannerScope, snapshot: WarehouseSnapshot ) -> frozenset[str]: relevant: set[CompiledObjectKey] = set(scope.selected_keys) frontier: list[CompiledObjectKey] = list(scope.selected_keys) @@ -354,7 +356,7 @@ def _new_model_names( def _metadata_jsons( - *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...] + *, runtime: PlannerRuntime, models: tuple[CompiledModelView, ...] ) -> dict[str, str]: function_hashes: dict[str, str] = build_function_local_hashes( functions=runtime.project.functions, dialect=runtime.project.sql_analysis_dialect @@ -375,7 +377,7 @@ def _origin_candidates( *, runtime: PlannerRuntime, state: MigrationStateInspection, - models: tuple[CompiledModel, ...], + models: tuple[CompiledModelView, ...], excluded: frozenset[tuple[str, str]], ) -> tuple[Fingerprint, ...]: project_model_names: frozenset[str] = frozenset(model.name for model in models) @@ -408,7 +410,7 @@ def _superseded(*, events: tuple[MigrationEvent, ...], location: CompiledRelatio def _match( *, - models: tuple[CompiledModel, ...], + models: tuple[CompiledModelView, ...], new_names: frozenset[str], seed_variants: tuple[dict[str, str], ...], candidates: tuple[Fingerprint, ...], @@ -430,7 +432,7 @@ def _match( variants: tuple[dict[str, str], ...] = tuple(dict(seed) for seed in seed_variants) matches: dict[str, Fingerprint] = {} claims: dict[str, list[str]] = defaultdict(list) - model: CompiledModel + model: CompiledModelView for model in models: if model.name not in new_names: continue @@ -469,7 +471,7 @@ def _match( def _variant_fingerprints( *, - model: CompiledModel, + model: CompiledModelView, metadata_json: str, variants: tuple[dict[str, str], ...], dialect: str | None, @@ -546,25 +548,25 @@ def _destination_fingerprints( return destinations -def _moves_history(*, model: CompiledModel, origin: Fingerprint) -> bool: +def _moves_history(*, model: CompiledModelView, origin: Fingerprint) -> bool: return _stores_history(model) and _stored_materialization(origin) in _HISTORY_MATERIALIZATIONS -def _renames_identity(*, model: CompiledModel, origin: Fingerprint) -> bool: +def _renames_identity(*, model: CompiledModelView, origin: Fingerprint) -> bool: return ( _hands_over_identity(model) and _stored_materialization(origin) in _IDENTITY_MATERIALIZATIONS ) -def _hands_over_identity(model: CompiledModel) -> bool: +def _hands_over_identity(model: CompiledModelView) -> bool: return ( get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) in _IDENTITY_MATERIALIZATIONS ) -def _stores_history(model: CompiledModel) -> bool: +def _stores_history(model: CompiledModelView) -> bool: return ( get_config_str(values=model.config.values, key=_MATERIALIZED_KEY) in _HISTORY_MATERIALIZATIONS diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/entry_fingerprints.py b/src/sqlbuild/compiler/planner/_helpers/migrations/entry_fingerprints.py index 36704edea2..665bcea14f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/entry_fingerprints.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/entry_fingerprints.py @@ -5,7 +5,7 @@ from collections.abc import Mapping from dataclasses import replace -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner.classes.migration_fingerprint_cache import ( MigrationFingerprintCache, ) @@ -16,7 +16,7 @@ def with_migration_fingerprints( *, entries: tuple[ModelPlanEntry, ...], - models_by_name: Mapping[str, CompiledModel], + models_by_name: Mapping[str, CompiledModelView], fingerprints: MigrationFingerprintCache, dialect: str | None, ) -> tuple[ModelPlanEntry, ...]: @@ -25,7 +25,7 @@ def with_migration_fingerprints( attached: list[ModelPlanEntry] = [] entry: ModelPlanEntry for entry in entries: - model: CompiledModel | None = models_by_name.get(entry.name) + model: CompiledModelView | None = models_by_name.get(entry.name) if ( model is None or entry.fingerprint_metadata_json is None diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py index 791b703601..154178c91b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/old_names.py @@ -8,8 +8,8 @@ from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import RelationType from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.models import CompiledModel, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.migrations.main._resolve_old_name_view_retention import ( resolve_old_name_view_retention, ) @@ -86,10 +86,10 @@ def plan_old_name_views( ) -> OldNameViewPlanning: """Decide old-name steps for selected models and collect recorded compatibility views.""" - models_by_name: dict[str, CompiledModel] = { + models_by_name: dict[str, CompiledModelView] = { model.name: model for model in runtime.project.models } - selected: tuple[CompiledModel, ...] = tuple( + selected: tuple[CompiledModelView, ...] = tuple( models_by_name[key.name] for key in scope.execution_order if key in scope.selected_keys @@ -138,7 +138,7 @@ def plan_old_name_views( if planned is not None: entries.append(planned) covered.add(entry.model_name) - model: CompiledModel + model: CompiledModelView for model in selected: if model.name in covered: continue @@ -164,13 +164,13 @@ def __init__( self, *, runtime: PlannerRuntime, - models_by_name: dict[str, CompiledModel], + models_by_name: dict[str, CompiledModelView], histories: tuple[OldNameViewHistory, ...], views: tuple[OldNameView, ...], column_entries: tuple[ColumnMigrationPlanEntry, ...], ) -> None: self._runtime: PlannerRuntime = runtime - self._models_by_name: dict[str, CompiledModel] = models_by_name + self._models_by_name: dict[str, CompiledModelView] = models_by_name self._histories: tuple[OldNameViewHistory, ...] = histories self._views: tuple[OldNameView, ...] = views self._column_entries: tuple[ColumnMigrationPlanEntry, ...] = column_entries @@ -183,7 +183,7 @@ def __init__( self._now: datetime = runtime.invocation_time def for_migration(self, entry: ModelMigrationPlanEntry) -> OldNameViewPlanEntry | None: - model: CompiledModel | None = self._models_by_name.get(entry.model_name) + model: CompiledModelView | None = self._models_by_name.get(entry.model_name) if model is None or entry.blocks_build: return None if _starts_move(entry): @@ -198,7 +198,7 @@ def for_migration(self, entry: ModelMigrationPlanEntry) -> OldNameViewPlanEntry return None if history is None else self._from_history(history=history, model=model) return None - def pending_for(self, model: CompiledModel) -> OldNameViewPlanEntry | None: + def pending_for(self, model: CompiledModelView) -> OldNameViewPlanEntry | None: history: OldNameViewHistory | None = self._newest_history( destination=migration_relation_for_location(model.destination), origin=None ) @@ -210,7 +210,7 @@ def pending_for(self, model: CompiledModel) -> OldNameViewPlanEntry | None: return planned def _new_move( - self, *, entry: ModelMigrationPlanEntry, model: CompiledModel + self, *, entry: ModelMigrationPlanEntry, model: CompiledModelView ) -> OldNameViewPlanEntry: retention: str | None = resolve_old_name_view_retention( config_values=model.config.values, project_retention=self._project_retention @@ -247,7 +247,7 @@ def _skip_reason( if retention is None: return "old_name_view false" old: MigrationRelation = migration_relation_for_location(origin) - reuser: CompiledModel | None = next( + reuser: CompiledModelView | None = next( ( model for model in self._models_by_name.values() @@ -264,7 +264,7 @@ def _skip_reason( return None def _from_history( - self, *, history: OldNameViewHistory, model: CompiledModel + self, *, history: OldNameViewHistory, model: CompiledModelView ) -> OldNameViewPlanEntry | None: """Resume a recorded move; its record proves SQLBuild owned the origin when it moved.""" @@ -325,7 +325,7 @@ def _newest_history( return None return max(matching, key=lambda item: (item.move.created_at, item.move.event_id)) - def _planned_aliases(self, model: CompiledModel) -> tuple[tuple[str, str], ...]: + def _planned_aliases(self, model: CompiledModelView) -> tuple[tuple[str, str], ...]: if _stores_history(model): return tuple( sorted( @@ -337,7 +337,7 @@ def _planned_aliases(self, model: CompiledModel) -> tuple[tuple[str, str], ...]: return declared_column_aliases(model) -def declared_column_aliases(model: CompiledModel) -> tuple[tuple[str, str], ...]: +def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: """Return old-to-new column names a table or view model declares with migrate_from.""" columns: tuple[SchemaColumn, ...] = ( @@ -378,7 +378,7 @@ def _skipped_resume( ) -def _stores_history(model: CompiledModel) -> bool: +def _stores_history(model: CompiledModelView) -> bool: return ( get_config_str(values=model.config.values, key="materialized") in _HISTORY_MATERIALIZATIONS ) @@ -398,7 +398,7 @@ def _skipped_entry(*, entry: ModelMigrationPlanEntry, reason: str) -> OldNameVie def _needed_schemas( *, runtime: PlannerRuntime, - selected: tuple[CompiledModel, ...], + selected: tuple[CompiledModelView, ...], snapshot: WarehouseSnapshot, migration_entries: tuple[ModelMigrationPlanEntry, ...], ) -> set[str]: @@ -429,7 +429,7 @@ def _schemas_to_read(*, snapshot: WarehouseSnapshot, needed: set[str]) -> set[st } -def _may_claim_view(*, model: CompiledModel, snapshot: WarehouseSnapshot) -> bool: +def _may_claim_view(*, model: CompiledModelView, snapshot: WarehouseSnapshot) -> bool: relation: RelationInfo | None = snapshot.existing_relations.get(model.name) return relation is not None and _is_view(relation) @@ -440,7 +440,7 @@ def _is_view(relation: RelationInfo) -> bool: def _claim_errors( *, - selected: Iterable[CompiledModel], + selected: Iterable[CompiledModelView], snapshot: WarehouseSnapshot, views: tuple[OldNameView, ...], now: datetime, @@ -448,7 +448,7 @@ def _claim_errors( """Refuse a model whose destination is a compatibility view still recorded at that name.""" errors: list[PlanWarning] = [] - model: CompiledModel + model: CompiledModelView for model in selected: if not _may_claim_view(model=model, snapshot=snapshot): continue @@ -563,7 +563,7 @@ def claim_message(*, model_name: str, view: OldNameView, now: datetime) -> str: def _recorded_view( - *, runtime: PlannerRuntime, history: OldNameViewHistory, models: Iterable[CompiledModel] + *, runtime: PlannerRuntime, history: OldNameViewHistory, models: Iterable[CompiledModelView] ) -> OldNameView: return OldNameView( destination_model=history.move.destination_model, diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py index 4b1638a0cc..18cd4c0b3a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/planning.py @@ -9,8 +9,8 @@ from sqlbuild.adapter.contract.models import ColumnInfo, MigrationStagePlan, RelationInfo from sqlbuild.adapter.contract.types import BuiltinAdapter, MigrationTransfer from sqlbuild.compiler.compile.constants import MIGRATE_FORCE_CONFIG_KEY, MIGRATE_FROM_CONFIG_KEY -from sqlbuild.compiler.compile.models import CompiledModel, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledRelationLocation +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.migrations.main._newest_event import newest_migration_event_mentioning @@ -87,14 +87,14 @@ def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationReq CompiledResourceType.MODEL ): continue - model: CompiledModel | None = scope.models_by_name.get(getattr(key, "name", "")) + model: CompiledModelView | None = scope.models_by_name.get(getattr(key, "name", "")) request: ModelMigrationRequest | None = None if model is None else _declared_request(model) if request is not None: requests.append(request) return tuple(requests) -def _declared_request(model: CompiledModel) -> ModelMigrationRequest | None: +def _declared_request(model: CompiledModelView) -> ModelMigrationRequest | None: raw_origin: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) if not isinstance(raw_origin, str): return None @@ -114,7 +114,7 @@ def _project_declarations( """Resolve every project model's migrate_from, selected or not.""" declarations: list[ModelMigrationDeclaration] = [] - model: CompiledModel + model: CompiledModelView for model in runtime.project.models: request: ModelMigrationRequest | None = _declared_request(model) if request is None: @@ -276,7 +276,7 @@ def _renamed_models( def _equivalent_definition( *, - model: CompiledModel, + model: CompiledModelView, handover: Fingerprint | None, destination_fingerprints: tuple[str, ...], dialect: str | None, @@ -373,7 +373,7 @@ def _resolve_named_origin( runtime: PlannerRuntime, state: MigrationStateInspection, ) -> tuple[CompiledRelationLocation, str | None]: - project_model: CompiledModel | None = next( + project_model: CompiledModelView | None = next( (model for model in runtime.project.models if model.name == name), None ) if project_model is not None: @@ -420,7 +420,7 @@ def _decide( state: MigrationStateInspection, snapshots_config: SnapshotsConfig, ) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: - model: CompiledModel = request.model + model: CompiledModelView = request.model destination: CompiledRelationLocation = model.destination if _relation_key(origin) == _relation_key(destination): raise PlannerInputError( @@ -522,7 +522,7 @@ def _decide( def _decide_rename( *, - model: CompiledModel, + model: CompiledModelView, base: ModelMigrationPlanEntry, newest: MigrationEvent | None, origin: CompiledRelationLocation, @@ -562,7 +562,7 @@ def _decide_rename( def _with_execution( - *, runtime: PlannerRuntime, model: CompiledModel, entry: ModelMigrationPlanEntry + *, runtime: PlannerRuntime, model: CompiledModelView, entry: ModelMigrationPlanEntry ) -> ModelMigrationPlanEntry: """Describe how the executor will stage and promote this migration.""" @@ -597,7 +597,7 @@ def _promotion(*, runtime: PlannerRuntime, decision: MigrationDecision) -> Migra def _done_handover_applies( - *, model: CompiledModel, event: MigrationEvent, snapshot: WarehouseSnapshot + *, model: CompiledModelView, event: MigrationEvent, snapshot: WarehouseSnapshot ) -> bool: own: Fingerprint | None = snapshot.fingerprints.models.get(model.name) return own is None or _utc(own.ts) < _utc(event.created_at) @@ -611,7 +611,7 @@ def _utc(value: datetime) -> datetime: def _done_handover( *, - model: CompiledModel, + model: CompiledModelView, event: MigrationEvent, snapshot: WarehouseSnapshot, origin_fingerprint: Fingerprint | None, @@ -624,7 +624,7 @@ def _done_handover( def _handover_fingerprint( - *, model: CompiledModel, fingerprint: Fingerprint | None, version_hash: str + *, model: CompiledModelView, fingerprint: Fingerprint | None, version_hash: str ) -> Fingerprint | None: if fingerprint is None: return None @@ -756,7 +756,7 @@ def redirected_cursor_snapshots( def _has_cursor(*, runtime: PlannerRuntime, model_name: str) -> bool: - model: CompiledModel | None = next( + model: CompiledModelView | None = next( (model for model in runtime.project.models if model.name == model_name), None ) return ( diff --git a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py index bb6a60c540..5ece1a1348 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/audit_entry.py @@ -10,11 +10,10 @@ AuditSeverity, ) from sqlbuild.compiler.compile.models import ( - CompiledAudit, CompiledObjectKey, CompiledRelationLocation, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView from sqlbuild.compiler.planner._helpers.output.audit_scheduling import ( reads_outside_target_lineage, resolve_attachment_kind, @@ -29,7 +28,7 @@ def plan_audit( *, - audit: CompiledAudit, + audit: CompiledAuditView, model_locations: dict[str, CompiledRelationLocation], seed_locations: dict[str, CompiledRelationLocation], source_map: dict[str, SourceEntry], diff --git a/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py b/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py index d8fb2dc0ad..e4851055be 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/audit_scheduling.py @@ -7,11 +7,11 @@ AuditRunScope, ) from sqlbuild.compiler.compile.models import ( - CompiledAudit, CompiledObjectKey, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, + CompiledAuditView, CompiledResourceType, ) from sqlbuild.compiler.planner._helpers.graph.core import expand_downstream, expand_upstream @@ -28,7 +28,7 @@ def resolve_attachment_kind( *, - audit: CompiledAudit, + audit: CompiledAuditView, downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], ) -> tuple[AuditAttachmentKind, str | None]: """Resolve audit attachment kind and attached target name; attached audits gate their target.""" @@ -46,7 +46,7 @@ def resolve_attachment_kind( def reads_outside_target_lineage( *, - audit: CompiledAudit, + audit: CompiledAuditView, upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], ) -> bool: """Return whether an attached model audit reads anything besides its target and upstream.""" @@ -84,7 +84,7 @@ def resolve_effective_run_scope( def _infer_singular_attachment( *, - audit: CompiledAudit, + audit: CompiledAuditView, downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], ) -> tuple[AuditAttachmentKind, str | None]: """Infer attachment for a singular audit from its refs and graph structure.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py index 44032415a4..ff55bfd3a3 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/microbatch_bounds.py @@ -7,10 +7,9 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._cursor_roles import resolve_cursor_input_roles from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompileSqlReference, ) +from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView from sqlbuild.compiler.planner._helpers.resolve.lineage import resolve_lineage_reference from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( aligned_timestamp_bounds, @@ -91,9 +90,9 @@ def cap_microbatch_bounds( def validate_capped_watermark_inputs( *, - model: CompiledModel, - models_by_name: dict[str, CompiledModel], - functions_by_name: dict[str, CompiledFunction] | None = None, + model: CompiledModelView, + models_by_name: dict[str, CompiledModelView], + functions_by_name: dict[str, CompiledFunctionView] | None = None, ) -> None: """Reject capped producers feeding microbatch watermark inputs.""" @@ -110,7 +109,7 @@ def validate_capped_watermark_inputs( models_by_name=models_by_name, functions_by_name=functions_by_name or {}, ) - producer: CompiledModel | None = _find_capped_producer_ancestor( + producer: CompiledModelView | None = _find_capped_producer_ancestor( reference=ref, models_by_name=models_by_name, functions_by_name=functions_by_name or {}, @@ -131,9 +130,9 @@ def validate_capped_watermark_inputs( def _find_capped_producer_ancestor( *, reference: CompileSqlReference, - models_by_name: dict[str, CompiledModel], - functions_by_name: dict[str, CompiledFunction], -) -> CompiledModel | None: + models_by_name: dict[str, CompiledModelView], + functions_by_name: dict[str, CompiledFunctionView], +) -> CompiledModelView | None: """Return the first capped microbatch model in a reference's upstream lineage.""" pending: list[CompileSqlReference] = [reference] @@ -145,7 +144,7 @@ def _find_capped_producer_ancestor( continue visited.add(identity) if current.ref_kind == SqlReferenceKind.REF: - producer: CompiledModel | None = models_by_name.get(current.ref_name) + producer: CompiledModelView | None = models_by_name.get(current.ref_name) if producer is None: continue action: MicrobatchLimitAction | None = _resolve_microbatch_limit_config( @@ -174,7 +173,7 @@ def _find_capped_producer_ancestor( SqlReferenceKind.UDF, SqlReferenceKind.TABLE_FUNCTION, }: - function: CompiledFunction | None = functions_by_name.get(current.ref_name) + function: CompiledFunctionView | None = functions_by_name.get(current.ref_name) if function is not None: pending.extend(function.references) return None diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 60df83ed6f..2b3b6655bf 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -14,18 +14,21 @@ from sqlbuild.compiler.compile.main._cursor_roles import resolve_cursor_input_roles from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledFunction, - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledRelationLocation, - CompiledSeed, CompilerDiagnostic, CompileSqlReference, CursorInputRoles, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType +from sqlbuild.compiler.compile.types import ( + AttachedAuditTargetKind, + CompiledAuditView, + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompiledSeedView, +) from sqlbuild.compiler.discovery.main.microbatch_guidance import microbatch_guidance from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.graph.source_load_nodes import build_source_load_map @@ -209,7 +212,7 @@ class _CustomMaterializationConfig: def build_planner_relations_context( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, scope: PlannerScope, @@ -306,7 +309,7 @@ def build_planner_relations_context( def _resolve_source_warehouse_columns( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, source_read_map: dict[str, SourceEntry], @@ -331,7 +334,7 @@ def _resolve_source_warehouse_columns( def plan_model( *, - model: CompiledModel, + model: CompiledModelView, snapshot: WarehouseSnapshot, adapter: BaseAdapter, context: ModelPlanContext, @@ -366,7 +369,7 @@ def plan_model( def build_plan_entries( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, scope: PlannerScope, snapshot: WarehouseSnapshot, @@ -422,7 +425,7 @@ def build_plan_entries( for key in scope.execution_order: if key not in scope.selected_keys or key.name not in resolved_actions.models: continue - model: CompiledModel | None = scope.models_by_name.get(key.name) + model: CompiledModelView | None = scope.models_by_name.get(key.name) resolved: ResolvedModelAction = resolved_actions.models[key.name] if model is None: continue @@ -675,7 +678,7 @@ def _apply_microbatch_limit( def plan_model_from_change( *, - model: CompiledModel, + model: CompiledModelView, snapshot: WarehouseSnapshot, adapter: BaseAdapter, context: ModelPlanContext, @@ -984,7 +987,7 @@ def plan_model_from_change( def resolve_cursor_overrides( *, - model: CompiledModel, + model: CompiledModelView, cursor_overrides: CursorOverrides | None, start_cursor_override: str | None, end_cursor_override: str | None, @@ -1038,7 +1041,7 @@ def resolve_cursor_overrides( return resolved_start, resolved_end -def _get_on_schema_change(model: CompiledModel) -> OnSchemaChange | None: +def _get_on_schema_change(model: CompiledModelView) -> OnSchemaChange | None: """Extract on_schema_change from model config.""" raw: object | None = model.config.values.get("on_schema_change") @@ -1049,7 +1052,7 @@ def _get_on_schema_change(model: CompiledModel) -> OnSchemaChange | None: raise PlannerInputError(f"model '{model.name}': unknown on_schema_change '{raw}'") -def extract_seed_columns(seed: CompiledSeed) -> tuple[ColumnInfo, ...]: +def extract_seed_columns(seed: CompiledSeedView) -> tuple[ColumnInfo, ...]: """Extract declared columns from seed schema entry.""" columns: list[ColumnInfo] = [] @@ -1060,7 +1063,7 @@ def extract_seed_columns(seed: CompiledSeed) -> tuple[ColumnInfo, ...]: return tuple(columns) -def _get_type_enforcement(model: CompiledModel) -> bool: +def _get_type_enforcement(model: CompiledModelView) -> bool: """Resolve whether type enforcement is active for a model.""" if model.schema_entry is not None and model.schema_entry.type_enforcement is not None: @@ -1068,7 +1071,7 @@ def _get_type_enforcement(model: CompiledModel) -> bool: return False -def _get_declared_columns(model: CompiledModel) -> tuple[ColumnInfo, ...]: +def _get_declared_columns(model: CompiledModelView) -> tuple[ColumnInfo, ...]: """Extract declared columns with types from schema entry.""" if model.schema_entry is None: @@ -1081,7 +1084,7 @@ def _get_declared_columns(model: CompiledModel) -> tuple[ColumnInfo, ...]: return tuple(columns) -def _get_contract_columns(model: CompiledModel) -> tuple[ColumnInfo, ...]: +def _get_contract_columns(model: CompiledModelView) -> tuple[ColumnInfo, ...]: if model.schema_entry is None: return () return tuple( @@ -1089,7 +1092,7 @@ def _get_contract_columns(model: CompiledModel) -> tuple[ColumnInfo, ...]: ) -def _get_unique_key(model: CompiledModel) -> tuple[str, ...]: +def _get_unique_key(model: CompiledModelView) -> tuple[str, ...]: """Extract unique_key from model config as a normalized tuple.""" raw: object | None = model.config.values.get("unique_key") @@ -1100,7 +1103,7 @@ def _get_unique_key(model: CompiledModel) -> tuple[str, ...]: return () -def _get_check_columns(model: CompiledModel) -> tuple[str, ...]: +def _get_check_columns(model: CompiledModelView) -> tuple[str, ...]: """Extract check_columns from model config as a normalized tuple.""" raw: object | None = model.config.values.get("check_columns") @@ -1112,7 +1115,7 @@ def _get_check_columns(model: CompiledModel) -> tuple[str, ...]: def _resolve_cursor_safety_configs( - *, model: CompiledModel, context: ModelPlanContext + *, model: CompiledModelView, context: ModelPlanContext ) -> tuple[FutureCursorsConfig | None, StartCursorsConfig | None]: return ( resolve_future_cursor_config(model=model, project_config=context.future_cursor_config), @@ -1120,7 +1123,7 @@ def _resolve_cursor_safety_configs( ) -def _resolve_model_plan_config(*, model: CompiledModel) -> _ModelPlanConfig: +def _resolve_model_plan_config(*, model: CompiledModelView) -> _ModelPlanConfig: return _ModelPlanConfig( incremental_strategy=get_config_str(values=model.config.values, key="incremental_strategy"), incremental_mode=get_config_str(values=model.config.values, key="incremental_mode"), @@ -1151,7 +1154,7 @@ def _resolve_model_plan_config(*, model: CompiledModel) -> _ModelPlanConfig: def _resolve_custom_materialization_config( - *, model: CompiledModel, materialization_type: MaterializationType + *, model: CompiledModelView, materialization_type: MaterializationType ) -> _CustomMaterializationConfig: if materialization_type != MaterializationType.CUSTOM: return _CustomMaterializationConfig(name=None, config={}, placeholders={}) @@ -1174,7 +1177,7 @@ def _resolve_custom_materialization_config( def _compute_microbatch_range( - *, model: CompiledModel, inputs: _MicrobatchRangeInputs + *, model: CompiledModelView, inputs: _MicrobatchRangeInputs ) -> CursorBounds | None: """Compute the real overall cursor range for microbatch batch splitting.""" @@ -1279,7 +1282,7 @@ def _compute_microbatch_range( def resolve_microbatch_lookback( - *, model: CompiledModel, effective_grain: str | None = None + *, model: CompiledModelView, effective_grain: str | None = None ) -> str | None: """Resolve explicit or default lookback for one microbatch model.""" @@ -1310,7 +1313,7 @@ def resolve_microbatch_lookback( def _compute_plan_cursor_bounds( *, - model: CompiledModel, + model: CompiledModelView, snapshot: WarehouseSnapshot, backfill: BackfillResult, full_refresh: bool, @@ -1397,7 +1400,7 @@ def _compute_plan_cursor_bounds( def _resolve_empty_cursor_inputs( *, - model: CompiledModel, + model: CompiledModelView, snapshot: WarehouseSnapshot, full_refresh: bool, runtime_owned_cursor_bounds: bool, @@ -1570,11 +1573,11 @@ def _build_logical_ddl_from_adapter( def _build_cursor_input_relations( *, - model: CompiledModel, + model: CompiledModelView, adapter: BaseAdapter, model_locations: dict[str, CompiledRelationLocation], - models_by_name: dict[str, CompiledModel], - functions_by_name: dict[str, CompiledFunction] | None = None, + models_by_name: dict[str, CompiledModelView], + functions_by_name: dict[str, CompiledFunctionView] | None = None, seed_locations: dict[str, CompiledRelationLocation], source_map: dict[str, SourceEntry], cursor_column: str | None, @@ -1599,7 +1602,7 @@ def _build_cursor_input_relations( models_by_name=models_by_name, functions_by_name=functions_by_name or {}, ) - producer_model: CompiledModel | None = ( + producer_model: CompiledModelView | None = ( models_by_name.get(ref.ref_name) if ref.ref_kind == SqlReferenceKind.REF else None ) relation: str | None = _resolve_cursor_input_relation( @@ -1671,10 +1674,10 @@ def _build_cursor_input_relations( def validate_source_cursor_input_columns( *, - model: CompiledModel, + model: CompiledModelView, cursor_column: str | None, - models_by_name: dict[str, CompiledModel] | None = None, - functions_by_name: dict[str, CompiledFunction] | None = None, + models_by_name: dict[str, CompiledModelView] | None = None, + functions_by_name: dict[str, CompiledFunctionView] | None = None, source_map: dict[str, SourceEntry] | None = None, source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]], ) -> None: @@ -1685,7 +1688,7 @@ def validate_source_cursor_input_columns( return cursor_roles: CursorInputRoles = resolve_cursor_input_roles(model=model) - effective_models_by_name: dict[str, CompiledModel] = models_by_name or {} + effective_models_by_name: dict[str, CompiledModelView] = models_by_name or {} effective_source_map: dict[str, SourceEntry] = source_map or {} authored_inputs: tuple[tuple[str, str], ...] = tuple( cursor_roles.filter_inputs.items() @@ -1700,7 +1703,7 @@ def validate_source_cursor_input_columns( functions_by_name=functions_by_name or {}, ) if ref.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModel | None = effective_models_by_name.get(ref.ref_name) + upstream_model: CompiledModelView | None = effective_models_by_name.get(ref.ref_name) if upstream_model is None: continue has_enforced_contract: bool = ( @@ -1749,7 +1752,7 @@ def validate_source_cursor_input_columns( ) -def _model_declared_column_names(model: CompiledModel) -> tuple[str, ...]: +def _model_declared_column_names(model: CompiledModelView) -> tuple[str, ...]: if model.schema_entry is None: return () return tuple(column.name for column in model.schema_entry.columns) @@ -1831,13 +1834,13 @@ def _cursor_input_kind( def _resolve_cursor_input_grain( *, ref: CompileSqlReference, - models_by_name: dict[str, CompiledModel], + models_by_name: dict[str, CompiledModelView], ) -> str | None: """Resolve timestamp grain metadata for a model-backed cursor input relation.""" if ref.ref_kind != SqlReferenceKind.REF: return None - upstream_model: CompiledModel | None = models_by_name.get(ref.ref_name) + upstream_model: CompiledModelView | None = models_by_name.get(ref.ref_name) if upstream_model is None: return None return get_config_str(values=upstream_model.config.values, key="cursor_grain") @@ -1859,7 +1862,7 @@ def scope_overlaps( def audit_is_selected( *, - audit: CompiledAudit, + audit: CompiledAuditView, selected_keys: frozenset[CompiledObjectKey], ) -> bool: """Return whether an audit belongs to the selected logical target scope.""" @@ -1875,13 +1878,13 @@ def audit_is_selected( def _selected_audit_relation_keys( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey], ) -> frozenset[CompiledObjectKey]: """Return relation keys needed to render selected audits without selecting those relations.""" relation_keys: set[CompiledObjectKey] = set() - audit: CompiledAudit + audit: CompiledAuditView for audit in project.audits: if audit_is_selected(audit=audit, selected_keys=selected_keys): relation_keys.update(audit.scope_deps) @@ -1897,7 +1900,7 @@ def build_model_materializations( def _rolling_window_bounds( - *, model: CompiledModel, invocation_time: datetime | None + *, model: CompiledModelView, invocation_time: datetime | None ) -> CursorBounds | None: grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") lookback: str | None = resolve_microbatch_lookback(model=model, effective_grain=grain) @@ -1938,7 +1941,9 @@ def _floor_utc_grain(*, moment: datetime, grain: str) -> datetime: return moment.replace(microsecond=0) -def _clamp_cursor_end(*, bounds: CursorBounds | None, model: CompiledModel) -> CursorBounds | None: +def _clamp_cursor_end( + *, bounds: CursorBounds | None, model: CompiledModelView +) -> CursorBounds | None: cursor_end: str | None = get_config_cursor_bound(values=model.config.values, key="cursor_end") cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") if bounds is None or cursor_end is None: diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py index fcb530f027..4094b07aad 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_output.py @@ -4,12 +4,14 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledFunction, CompiledObjectKey, - CompiledProject, CompiledSqlTest, ) +from sqlbuild.compiler.compile.types import ( + CompiledAuditView, + CompiledFunctionView, + CompiledProjectView, +) from sqlbuild.compiler.discovery.models import ( DiscoveredHookFunction, DiscoveredLoaderFunction, @@ -18,10 +20,8 @@ PythonHookEntry, ) from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.graph.main._build_lineage_downstream_deps import ( - build_lineage_downstream_deps, -) -from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import build_lineage_upstream_deps +from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views +from sqlbuild.compiler.graph.models import LineageGraphViews from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_context, ) @@ -67,7 +67,7 @@ def build_plan_output( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, scope: PlannerScope, snapshot: WarehouseSnapshot, @@ -204,7 +204,7 @@ def _resolve_seed_plan_reason( def _build_provider_usages( *, - project: CompiledProject, + project: CompiledProjectView, model_entries: tuple[ModelPlanEntry, ...], source_load_entries: tuple[SourceLoadPlanEntry, ...], ) -> tuple[PlanProviderUsage, ...]: @@ -309,7 +309,7 @@ def _to_plan_provider_usages( def _validate_skipped_intermediate_loader_targets( *, - project: CompiledProject, + project: CompiledProjectView, snapshot: WarehouseSnapshot, relations: PlannerRelationsContext, source_load_entries: tuple[SourceLoadPlanEntry, ...], @@ -347,7 +347,7 @@ def _validate_skipped_intermediate_loader_targets( def _build_function_entries( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, snapshot: WarehouseSnapshot, relations: PlannerRelationsContext, @@ -355,7 +355,7 @@ def _build_function_entries( selected_keys: frozenset[CompiledObjectKey], ) -> list[FunctionPlanEntry]: entries: list[FunctionPlanEntry] = [] - function: CompiledFunction + function: CompiledFunctionView for function in project.functions: if function.key not in selected_keys: continue @@ -379,7 +379,7 @@ def _build_function_entries( def plan_function( *, - function: CompiledFunction, + function: CompiledFunctionView, adapter: BaseAdapter, relations: PlannerRelationsContext, fingerprint_query_sql: str, @@ -420,23 +420,29 @@ def plan_function( def build_selected_audit_entries( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, scope: PlannerScope, relations: PlannerRelationsContext, model_materializations: dict[str, str], ) -> list[AuditPlanEntry]: entries: list[AuditPlanEntry] = [] + selected_audits: tuple[CompiledAuditView, ...] = tuple( + audit + for audit in project.audits + if audit_is_selected(audit=audit, selected_keys=scope.selected_keys) + ) + if not selected_audits: + return entries + lineage: LineageGraphViews = project_lineage_views(project) lineage_upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_upstream_deps(project) + lineage.upstream_deps ) lineage_downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_downstream_deps(lineage_upstream_deps) + lineage.downstream_deps ) - audit: CompiledAudit - for audit in project.audits: - if not audit_is_selected(audit=audit, selected_keys=scope.selected_keys): - continue + audit: CompiledAuditView + for audit in selected_audits: entries.append( plan_audit( audit=audit, @@ -454,7 +460,7 @@ def build_selected_audit_entries( def plan_selected_sql_tests( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, selected_keys: frozenset[CompiledObjectKey], sql_test_selection: SqlTestSelection, @@ -495,7 +501,7 @@ def sql_test_is_selected( def build_selected_test_entries( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, selected_keys: frozenset[CompiledObjectKey], sql_test_selection: SqlTestSelection, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index dd11f37608..59f9967230 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -2,9 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import ( - CompiledModel, -) +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.constants import ( PROJECT_CONFIG_FILENAME, QUERY_CHANGE_TRACKING_SETTING_KEY, @@ -49,7 +47,7 @@ def resolve_model_plan_action( *, - model: CompiledModel, + model: CompiledModelView, change_result: ChangeDetectionResult, full_refresh: bool, ) -> tuple[PlanAction, PlanReason]: @@ -264,7 +262,7 @@ def build_model_warnings( return tuple(warnings) -def _is_disabled(model: CompiledModel) -> bool: +def _is_disabled(model: CompiledModelView) -> bool: """Check whether a model is explicitly disabled.""" raw: object | None = model.config.values.get("enabled") @@ -329,7 +327,7 @@ def _table_action( def _incremental_action( *, - model: CompiledModel, + model: CompiledModelView, change_result: ChangeDetectionResult, ) -> tuple[PlanAction, PlanReason]: """Determine action for an incremental materialization.""" @@ -379,7 +377,7 @@ def _describe_finding(finding: SchemaFinding) -> str: return f"{finding.kind.value}" -def get_materialization_type(model: CompiledModel) -> MaterializationType: +def get_materialization_type(model: CompiledModelView) -> MaterializationType: """Extract materialization type from model config.""" raw: object | None = model.config.values.get("materialized") diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py b/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py index 7c20e68fa7..fc1eb330af 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/buildability.py @@ -2,7 +2,8 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.graph.buildability import ( check_buildability, missing_upstream_message, @@ -31,7 +32,7 @@ def check_selected_scope_buildability( ) -> None: """Raise a planner input error when selected upstream inputs are missing.""" - project: CompiledProject = runtime.project + project: CompiledProjectView = runtime.project snapshot: WarehouseSnapshot = warehouse.snapshot external_seed_keys: frozenset[CompiledObjectKey] = frozenset( seed.key for seed in project.seeds if seed.external diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py b/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py index 55a5a21a52..88e57852f1 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/full_refresh.py @@ -2,14 +2,14 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.planner.constants import FULL_REFRESH_DISABLED_REBUILD_CODE from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ChangeDetectionResult from sqlbuild.compiler.planner.types import ChangeKind -def resolve_model_full_refresh(*, model: CompiledModel, cli_full_refresh: bool) -> bool: +def resolve_model_full_refresh(*, model: CompiledModelView, cli_full_refresh: bool) -> bool: """Resolve dbt-compatible nullable model full-refresh semantics.""" configured: object | None = model.config.values.get("full_refresh") @@ -17,7 +17,7 @@ def resolve_model_full_refresh(*, model: CompiledModel, cli_full_refresh: bool) def effectively_full_refreshed_model_names( - *, project: CompiledProject, cli_full_refresh: bool + *, project: CompiledProjectView, cli_full_refresh: bool ) -> frozenset[str]: """Return model names that are effectively full refreshed for this invocation.""" @@ -48,7 +48,7 @@ def describe_full_rebuild_cause(*, change_result: ChangeDetectionResult, full_re def check_full_rebuild_allowed( - *, model: CompiledModel, change_result: ChangeDetectionResult, full_rebuild_cause: str + *, model: CompiledModelView, change_result: ChangeDetectionResult, full_rebuild_cause: str ) -> None: """Refuse a full rebuild of a full_refresh false model unless its own SQL or first run asks.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py index 5b0595530e..a7e2a38b40 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/identities.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/identities.py @@ -4,8 +4,8 @@ from dataclasses import replace -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.planner._helpers.changes.detect import detect_model_changes_in_scope from sqlbuild.compiler.planner._helpers.identity.direct import ( build_direct_model_version_identities, @@ -22,7 +22,7 @@ def build_planner_identity_context( *, - project: CompiledProject, + project: CompiledProjectView, scopes: PlannerScopeResolution, include_stale_warning_identities: bool = True, ) -> PlannerIdentityContext: @@ -53,7 +53,7 @@ def build_planner_identity_context( def detect_stale_warning_changes( *, - project: CompiledProject, + project: CompiledProjectView, scopes: PlannerScopeResolution, snapshot: WarehouseSnapshot, identities: PlannerIdentityContext, diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py index 9842ce637e..8490daf239 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/retention.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/retention.py @@ -14,8 +14,7 @@ RetentionState, ) from sqlbuild.adapter.contract.types import BuiltinAdapter, RetentionChangePhase, RetentionScope -from sqlbuild.compiler.compile.models import CompiledModel -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( PlannerRuntime, @@ -46,10 +45,10 @@ def plan_table_types( selected_names: frozenset[str] = frozenset( key.name for key in scope.selected_keys if key.resource_type == CompiledResourceType.MODEL ) - selected_models: tuple[CompiledModel, ...] = tuple( + selected_models: tuple[CompiledModelView, ...] = tuple( model for model in runtime.project.models if model.name in selected_names ) - declared_models: tuple[CompiledModel, ...] = tuple( + declared_models: tuple[CompiledModelView, ...] = tuple( model for model in selected_models if model.config.table_type.declared ) if declared_models and runtime.adapter.adapter_name != BuiltinAdapter.SNOWFLAKE: @@ -123,7 +122,7 @@ def plan_retention( selected_names: frozenset[str] = frozenset( key.name for key in scope.selected_keys if key.resource_type == CompiledResourceType.MODEL ) - selected_models: tuple[CompiledModel, ...] = tuple( + selected_models: tuple[CompiledModelView, ...] = tuple( model for model in runtime.project.models if model.name in selected_names @@ -147,7 +146,7 @@ def plan_retention( def _plan_relation_retention( - *, runtime: PlannerRuntime, warehouse: PlannerWarehouseState, model: CompiledModel + *, runtime: PlannerRuntime, warehouse: PlannerWarehouseState, model: CompiledModelView ) -> tuple[RetentionPlanEntry, ...]: retention: ResolvedTimeTravelRetention = model.config.time_travel_retention if retention.desired_days is None or retention.source is None: @@ -262,7 +261,7 @@ def _plan_relation_retention( def _plan_bigquery_retention( - *, runtime: PlannerRuntime, models: tuple[CompiledModel, ...] + *, runtime: PlannerRuntime, models: tuple[CompiledModelView, ...] ) -> tuple[RetentionPlanEntry, ...]: target: TargetConfig = effective_target_config(runtime=runtime) if not target.owns_time_travel_retention_namespace: @@ -270,8 +269,8 @@ def _plan_bigquery_retention( "BigQuery managed time_travel_retention requires target " "owns_time_travel_retention_namespace = true" ) - all_managed: dict[tuple[str | None, str], list[CompiledModel]] = defaultdict(list) - model: CompiledModel + all_managed: dict[tuple[str | None, str], list[CompiledModelView]] = defaultdict(list) + model: CompiledModelView for model in runtime.project.models: retention: ResolvedTimeTravelRetention = model.config.time_travel_retention if ( @@ -286,7 +285,7 @@ def _plan_bigquery_retention( } entries: list[RetentionPlanEntry] = [] for database, schema in sorted(selected_scopes, key=lambda item: (item[0] or "", item[1])): - scope_models: list[CompiledModel] = all_managed[(database, schema)] + scope_models: list[CompiledModelView] = all_managed[(database, schema)] desired_values: set[int] = { desired for item in scope_models diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py b/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py index 21b80bec6b..5ca65757d0 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/scopes.py @@ -4,7 +4,8 @@ from dataclasses import replace -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many from sqlbuild.compiler.planner._helpers.graph.scope import build_planner_scope from sqlbuild.compiler.planner.models import ( @@ -17,7 +18,7 @@ def resolve_planner_scopes( *, - project: CompiledProject, + project: CompiledProjectView, selection: PlannerSelection, policies: PlannerPolicies, ) -> PlannerScopeResolution: diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py index cc86a108fa..2853d9a560 100644 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py +++ b/src/sqlbuild/compiler/planner/_helpers/pruning/selection_staleness.py @@ -2,8 +2,8 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.pruning.selection_classifier import ( classify_selection_staleness_warnings, @@ -201,7 +201,7 @@ def _model_own_identity_changed( model_changes_complete or selected_change.change_kind in _OWN_CHANGE_KINDS ): return selected_change.change_kind in _OWN_CHANGE_KINDS - model: CompiledModel | None = original_scope.models_by_name.get(model_name) + model: CompiledModelView | None = original_scope.models_by_name.get(model_name) if model is None: return False change: ChangeDetectionResult = detect_model_changes( diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/config.py b/src/sqlbuild/compiler/planner/_helpers/resolve/config.py index 74652535a9..09533b960d 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/config.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/config.py @@ -2,12 +2,12 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner.types import IncrementalStrategy from sqlbuild.spec.contracts.main.get_config_str import get_config_str -def get_config_append_cursor_inclusive(model: CompiledModel) -> bool: +def get_config_append_cursor_inclusive(model: CompiledModelView) -> bool: """Return effective append cursor lower-bound inclusivity.""" strategy: str | None = get_config_str(values=model.config.values, key="incremental_strategy") diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py index 9ddbbc9cca..3efc3ffa7b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor.py @@ -7,7 +7,7 @@ from dataclasses import replace from datetime import datetime -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( CursorBounds, @@ -496,14 +496,14 @@ def _parse_required(*, value: str, cursor_type: str | None) -> CursorScalar: def type_cursor_overrides( *, cursor_overrides: CursorOverrides | None, - selected_models: Iterable[CompiledModel], + selected_models: Iterable[CompiledModelView], ) -> CursorOverrides | None: """Type `--start-cursor`/`--end-cursor` from the selected cursor models, else the literal.""" if cursor_overrides is None or not cursor_overrides.has_untyped: return cursor_overrides model_names_by_type: dict[CursorType, list[str]] = {} - model: CompiledModel + model: CompiledModelView for model in selected_models: cursor_type: CursorType | None = _model_cursor_type(model=model) if cursor_type is not None: @@ -529,7 +529,7 @@ def type_cursor_overrides( return cursor_overrides.typed_as(cursor_type=resolved_type) -def _model_cursor_type(*, model: CompiledModel) -> CursorType | None: +def _model_cursor_type(*, model: CompiledModelView) -> CursorType | None: materialized: str | None = get_config_str(values=model.config.values, key="materialized") cursor_column: str | None = get_config_str(values=model.config.values, key="cursor") if materialized != MaterializationType.INCREMENTAL or cursor_column is None: diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py index 2a8589bd47..f1885743da 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_policies.py @@ -2,14 +2,14 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.spec.contracts.constants import CURSOR_POLICY_DISABLED from sqlbuild.spec.contracts.models import FutureCursorsConfig, StartCursorsConfig from sqlbuild.spec.contracts.types import FutureCursorAction def resolve_start_cursor_config( - *, model: CompiledModel, project_config: StartCursorsConfig | None + *, model: CompiledModelView, project_config: StartCursorsConfig | None ) -> StartCursorsConfig | None: """Resolve automatic-start policy with model config taking precedence.""" @@ -26,7 +26,7 @@ def resolve_start_cursor_config( def resolve_future_cursor_config( - *, model: CompiledModel, project_config: FutureCursorsConfig | None + *, model: CompiledModelView, project_config: FutureCursorsConfig | None ) -> FutureCursorsConfig | None: """Resolve future-end policy with model config taking precedence.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py b/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py index 908e9c46f9..1e3929e4d5 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/lineage.py @@ -3,20 +3,19 @@ from __future__ import annotations from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompileSqlReference, ) +from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.references.types import SqlReferenceKind def resolve_lineage_reference( *, - model: CompiledModel, + model: CompiledModelView, input_name: str, - models_by_name: dict[str, CompiledModel], - functions_by_name: dict[str, CompiledFunction], + models_by_name: dict[str, CompiledModelView], + functions_by_name: dict[str, CompiledFunctionView], ) -> CompileSqlReference: """Resolve one named input through model and SQL function references.""" @@ -31,14 +30,16 @@ def resolve_lineage_reference( continue visited.add(identity) if reference.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModel | None = models_by_name.get(reference.ref_name) + upstream_model: CompiledModelView | None = models_by_name.get(reference.ref_name) if upstream_model is not None: pending.extend(upstream_model.references) elif reference.ref_kind in { SqlReferenceKind.UDF, SqlReferenceKind.TABLE_FUNCTION, }: - upstream_function: CompiledFunction | None = functions_by_name.get(reference.ref_name) + upstream_function: CompiledFunctionView | None = functions_by_name.get( + reference.ref_name + ) if upstream_function is not None: pending.extend(upstream_function.references) raise PlannerInputError( diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py index c15c31ce09..365cff4e0b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/refs.py @@ -10,11 +10,13 @@ resolve_relation_location_qualified_name, ) from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompiledObjectKey, CompiledRelationLocation, - CompiledSeed, +) +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledSeedView, ) from sqlbuild.compiler.planner.constants import ( SQL_ALIAS_BOUNDARY_CHARACTERS, @@ -26,7 +28,7 @@ from sqlbuild.compiler.references.main._quoted_reference_call_pattern import ( quoted_reference_call_pattern, ) -from sqlbuild.compiler.references.main.reference_call_prefix_pattern_text import ( +from sqlbuild.compiler.references.main._reference_call_prefix_pattern_text import ( reference_call_prefix_pattern_text, ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind @@ -366,7 +368,7 @@ def _has_following_alias(*, sql: str, start: int) -> bool: def build_model_locations( - models: tuple[CompiledModel, ...], + models: tuple[CompiledModelView, ...], ) -> dict[str, CompiledRelationLocation]: """Build a lookup of model name to compiled relation location.""" @@ -374,7 +376,7 @@ def build_model_locations( def build_seed_locations( - seeds: tuple[CompiledSeed, ...], + seeds: tuple[CompiledSeedView, ...], ) -> dict[str, CompiledRelationLocation]: """Build a lookup of seed name to compiled relation location.""" @@ -382,7 +384,7 @@ def build_seed_locations( def build_function_locations( - functions: tuple[CompiledFunction, ...], + functions: tuple[CompiledFunctionView, ...], ) -> dict[str, CompiledRelationLocation]: """Build a lookup of function name to compiled relation location.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index 0d747a75c8..7f78c47c09 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -10,11 +10,10 @@ from sqlbuild.compiler.compile.main._cursor_roles import resolve_cursor_input_roles from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompiledRelationLocation, CursorInputRoles, ) +from sqlbuild.compiler.compile.types import CompiledFunctionView, CompiledModelView from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.planner._helpers.output.inclusive_cursor_end import ( resolve_bounded_cursor_override, @@ -75,7 +74,7 @@ def resolve_model_sql( *, adapter: BaseAdapter, - model: CompiledModel, + model: CompiledModelView, snapshot: WarehouseSnapshot, context: ModelPlanContext, backfill: BackfillResult, @@ -185,7 +184,7 @@ def _unresolved_cursor_bounds_error( def resolve_function_sql( *, adapter: BaseAdapter, - function: CompiledFunction, + function: CompiledFunctionView, model_locations: dict[str, CompiledRelationLocation], seed_locations: dict[str, CompiledRelationLocation], function_locations: dict[str, CompiledRelationLocation], @@ -214,7 +213,7 @@ def resolve_function_sql( def _compute_model_cursor_bounds( *, - model: CompiledModel, + model: CompiledModelView, snapshot: WarehouseSnapshot, backfill: BackfillResult, full_refresh: bool, @@ -347,7 +346,7 @@ def _compute_model_cursor_bounds( ) -def _destination_exists(*, model: CompiledModel, snapshot: WarehouseSnapshot) -> bool: +def _destination_exists(*, model: CompiledModelView, snapshot: WarehouseSnapshot) -> bool: """Return whether the model's resolved destination, alias included, exists in full identity.""" relation: RelationInfo | None = snapshot.existing_relations.get(model.name) @@ -358,7 +357,7 @@ def _destination_exists(*, model: CompiledModel, snapshot: WarehouseSnapshot) -> ) -def _history_redirected(*, model: CompiledModel, cursor_snapshot: ModelCursorSnapshot) -> bool: +def _history_redirected(*, model: CompiledModelView, cursor_snapshot: ModelCursorSnapshot) -> bool: """Return whether the model's cursor history lives in a relation other than its destination.""" return ( @@ -369,7 +368,7 @@ def _history_redirected(*, model: CompiledModel, cursor_snapshot: ModelCursorSna def resolve_model_hook_entries( *, - model: CompiledModel, + model: CompiledModelView, adapter: BaseAdapter, context: ModelPlanContext, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, diff --git a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py index da3a724b58..d5320eac16 100644 --- a/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py +++ b/src/sqlbuild/compiler/planner/_helpers/reuse/clone.py @@ -6,31 +6,22 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledRelationLocation, - CompiledSource, ) -from sqlbuild.compiler.graph.main._build_lineage_downstream_deps import ( - build_lineage_downstream_deps, -) -from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import ( - build_lineage_upstream_deps, +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledSourceView, ) +from sqlbuild.compiler.graph.main._native_project_graph import build_native_project_graph from sqlbuild.compiler.planner._helpers.graph.core import ( build_downstream_deps, build_execution_upstream_deps, topologically_order_keys, ) -from sqlbuild.compiler.planner._helpers.graph.selector_indexes import ( - build_model_path_index_impl as build_model_path_index, -) -from sqlbuild.compiler.planner._helpers.graph.selector_indexes import ( - build_model_tag_index_impl as build_model_tag_index, -) -from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_selectors +from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_graph_selectors from sqlbuild.compiler.planner._helpers.identity.functions import ( build_compiled_function_fingerprint_sql, ) @@ -67,7 +58,7 @@ def build_clone_plan_output( *, - project: CompiledProject, + project: CompiledProjectView, select: tuple[str, ...], exclude: tuple[str, ...], ) -> PlanOutput: @@ -77,23 +68,8 @@ def build_clone_plan_output( downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( upstream_deps ) - all_keys: dict[str, CompiledObjectKey] = { - **{model.name: model.key for model in project.models}, - **{source.name: source.key for source in project.sources}, - **{seed.name: seed.key for seed in project.seeds}, - **{function.name: function.key for function in project.functions}, - } - lineage_upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_lineage_upstream_deps(project) - ) - selected_keys: frozenset[CompiledObjectKey] = resolve_selectors( - select=select, - exclude=exclude, - all_keys=all_keys, - upstream=lineage_upstream, - downstream=build_lineage_downstream_deps(lineage_upstream), - tag_index=build_model_tag_index(project), - path_index=build_model_path_index(project), + selected_keys: frozenset[CompiledObjectKey] = resolve_graph_selectors( + graph=build_native_project_graph(project), select=select, exclude=exclude ) return PlanOutput( execution_order=topologically_order_keys(upstream=upstream_deps), @@ -105,13 +81,13 @@ def build_clone_plan_output( def build_clone_model_entries( *, - project: CompiledProject, + project: CompiledProjectView, plan: PlanOutput, adapter: BaseAdapter, connection: Any, - source_project: CompiledProject | None = None, + source_project: CompiledProjectView | None = None, ) -> tuple[ModelPlanEntry, ...]: - effective_source_project: CompiledProject = source_project or project + effective_source_project: CompiledProjectView = source_project or project model_locations: dict[str, CompiledRelationLocation] = build_model_locations(project.models) seed_locations: dict[str, CompiledRelationLocation] = build_seed_locations(project.seeds) function_locations: dict[str, CompiledRelationLocation] = build_function_locations( @@ -126,7 +102,7 @@ def build_clone_model_entries( connection=connection, ) entries_by_key: dict[CompiledObjectKey, ModelPlanEntry] = {} - model: CompiledModel + model: CompiledModelView for model in project.models: if model.key not in plan.selected_keys or is_disabled(model): continue @@ -172,12 +148,12 @@ def build_clone_model_entries( def build_clone_source_entries( *, - project: CompiledProject, + project: CompiledProjectView, plan: PlanOutput, adapter: BaseAdapter, ) -> tuple[CloneSourcePlanEntry, ...]: entries_by_key: dict[CompiledObjectKey, CloneSourcePlanEntry] = {} - source: CompiledSource + source: CompiledSourceView for source in project.sources: source_entry: SourceEntry = source.source_entry if ( @@ -207,14 +183,14 @@ def build_clone_source_entries( def build_clone_function_entries( *, - project: CompiledProject, + project: CompiledProjectView, plan: PlanOutput, adapter: BaseAdapter, connection: Any, - source_project: CompiledProject | None = None, + source_project: CompiledProjectView | None = None, ) -> tuple[FunctionPlanEntry, ...]: """Build destination-resolved function definitions for clone recreation.""" - effective_source_project: CompiledProject = source_project or project + effective_source_project: CompiledProjectView = source_project or project model_locations: dict[str, CompiledRelationLocation] = build_model_locations(project.models) seed_locations: dict[str, CompiledRelationLocation] = build_seed_locations(project.seeds) function_locations: dict[str, CompiledRelationLocation] = build_function_locations( @@ -229,7 +205,7 @@ def build_clone_function_entries( connection=connection, ) entries_by_key: dict[CompiledObjectKey, FunctionPlanEntry] = {} - function: CompiledFunction + function: CompiledFunctionView for function in project.functions: if function.key not in plan.selected_keys: continue @@ -264,7 +240,7 @@ def build_clone_function_entries( def build_origin_source_entries( *, - project: CompiledProject, + project: CompiledProjectView, selected_names: frozenset[str], adapter: BaseAdapter, ) -> tuple[CloneSourcePlanEntry, ...]: @@ -283,11 +259,11 @@ def build_origin_source_entries( def build_origin_model_entries( *, - project: CompiledProject, + project: CompiledProjectView, selected_names: frozenset[str], ) -> tuple[ModelPlanEntry, ...]: entries: list[ModelPlanEntry] = [] - model: CompiledModel + model: CompiledModelView for model in project.models: if model.name not in selected_names or is_disabled(model): continue @@ -315,7 +291,7 @@ def build_origin_model_entries( def build_clone_seed_entries( *, - project: CompiledProject, + project: CompiledProjectView, plan: PlanOutput, ) -> tuple[SeedPlanEntry, ...]: return tuple( @@ -334,7 +310,7 @@ def build_clone_seed_entries( def build_origin_seed_entries( *, - project: CompiledProject, + project: CompiledProjectView, selected_names: frozenset[str], ) -> tuple[SeedPlanEntry, ...]: return tuple( @@ -351,6 +327,6 @@ def build_origin_seed_entries( ) -def is_disabled(model: CompiledModel) -> bool: +def is_disabled(model: CompiledModelView) -> bool: raw: object | None = model.config.values.get("enabled") return isinstance(raw, bool) and not raw diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py b/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py index 3d8c00965e..4a762c8440 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py @@ -4,10 +4,9 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, CompiledSqlScenario, ) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner._helpers.scenario.artifacts import ( build_scenario_hash_index, @@ -116,12 +115,12 @@ def _scenario_artifacts(graph_plan: ScenarioGraphPlan) -> tuple[ScenarioArtifact def _scenario_namespace( - *, project: CompiledProject, graph_plan: ScenarioGraphPlan + *, project: CompiledProjectView, graph_plan: ScenarioGraphPlan ) -> tuple[str | None, str | None]: - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} model_name: str for model_name in (*graph_plan.target_model_names, *graph_plan.model_names): - model: CompiledModel | None = models_by_name.get(model_name) + model: CompiledModelView | None = models_by_name.get(model_name) if model is not None: return model.destination.database, model.destination.schema return None, None diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py b/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py index 38ddc0e397..543cb3791b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/graph.py @@ -5,12 +5,14 @@ from dataclasses import dataclass from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledSqlScenario, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.planner.constants import SCENARIO_PLAN_GRAPH_VALIDATION from sqlbuild.compiler.planner.models import PlanWarning, ScenarioGraphPlan from sqlbuild.compiler.planner.types import WarningSeverity @@ -32,13 +34,13 @@ class _UpstreamRequirements: def plan_scenario_graph( *, scenario: CompiledSqlScenario, - project: CompiledProject, + project: CompiledProjectView, sql_analysis_enabled: bool = True, sql_analysis_dialect: str | None = None, ) -> tuple[ScenarioGraphPlan, tuple[PlanWarning, ...]]: """Infer scenario target models, build closure, and required fixtures.""" - model_map: dict[str, CompiledModel] = {model.name: model for model in project.models} + model_map: dict[str, CompiledModelView] = {model.name: model for model in project.models} source_names: frozenset[str] = frozenset(source.name for source in project.sources) seed_names: frozenset[str] = frozenset(seed.name for seed in project.seeds) del sql_analysis_enabled, sql_analysis_dialect @@ -128,7 +130,7 @@ def _declared_name_warnings( scenario: CompiledSqlScenario, target_model_names: tuple[str, ...], assertion_target_names: tuple[str, ...], - model_map: dict[str, CompiledModel], + model_map: dict[str, CompiledModelView], source_names: frozenset[str], seed_names: frozenset[str], ) -> tuple[PlanWarning, ...]: @@ -198,7 +200,7 @@ def _collect_upstream_requirements( *, target_model_names: tuple[str, ...], scenario_name: str, - model_map: dict[str, CompiledModel], + model_map: dict[str, CompiledModelView], ref_fixture_names: frozenset[str], ) -> _UpstreamRequirements: """Walk target model upstreams and collect required fixtures and functions.""" @@ -221,7 +223,7 @@ def _walk( if model_name in requirements.model_names: return requirements - model: CompiledModel | None = model_map.get(model_name) + model: CompiledModelView | None = model_map.get(model_name) if model is None: return requirements requirements = _UpstreamRequirements( diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py index 818139b9a6..b5d053ff31 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py @@ -16,16 +16,19 @@ ) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledRelationLocation, - CompiledSeed, CompiledSqlScenario, CompileSqlScenarioCte, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompiledSeedView, +) +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_completion, ) @@ -73,7 +76,7 @@ FixtureKey, ScenarioArtifactKind, ) -from sqlbuild.compiler.references.main.reference_call_prefix_pattern_text import ( +from sqlbuild.compiler.references.main._reference_call_prefix_pattern_text import ( reference_call_prefix_pattern_text, ) from sqlbuild.compiler.references.main.render_source_relation import render_source_relation @@ -99,7 +102,7 @@ def build_scenario_relation_plan( *, - project: CompiledProject, + project: CompiledProjectView, graph_plan: ScenarioGraphPlan, relation_map: ScenarioRelationMap, render_qualified_name: Callable[..., str | None], @@ -313,7 +316,7 @@ def _replace_relation_markers( def build_scenario_execution_plan( *, scenario: CompiledSqlScenario, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, graph_plan: ScenarioGraphPlan, relation_plan: ScenarioRelationPlan, @@ -328,8 +331,8 @@ def build_scenario_execution_plan( effective_source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] = ( source_warehouse_columns or {} ) - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} - functions_by_key: dict[CompiledObjectKey, CompiledFunction] = { + models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} + functions_by_key: dict[CompiledObjectKey, CompiledFunctionView] = { function.key: function for function in project.functions } function_locations: dict[str, CompiledRelationLocation] = build_function_locations( @@ -406,7 +409,7 @@ def build_scenario_execution_plan( warnings: list[PlanWarning] = [] key: CompiledObjectKey for key in ordered_model_keys: - model: CompiledModel = models_by_name[key.name] + model: CompiledModelView = models_by_name[key.name] scenario_target: CompiledRelationLocation = _required_target( targets=relation_plan.model_locations, name=model.name, @@ -419,7 +422,11 @@ def build_scenario_execution_plan( entry: ModelPlanEntry model_warnings: tuple[PlanWarning, ...] entry, model_warnings = plan_model( - model=replace(model, destination=scenario_target, query_sql=scenario_query_sql), + model=replace( + compiled_model_record(model), + destination=scenario_target, + query_sql=scenario_query_sql, + ), snapshot=effective_snapshot, adapter=adapter, context=ModelPlanContext( @@ -484,7 +491,7 @@ def build_scenario_execution_plan( def _build_scenario_function_entry( *, - function: CompiledFunction, + function: CompiledFunctionView, adapter: BaseAdapter, relation_plan: ScenarioRelationPlan, function_locations: dict[str, CompiledRelationLocation], @@ -698,20 +705,20 @@ def _apply_fixture_overrides( def build_scenario_seed_entries( *, - project: CompiledProject, + project: CompiledProjectView, graph_plan: ScenarioGraphPlan, relation_plan: ScenarioRelationPlan, ) -> tuple[SeedPlanEntry, ...]: """Build project seed load entries for required seeds not overridden by fixtures.""" - seeds_by_name: dict[str, CompiledSeed] = {seed.name: seed for seed in project.seeds} + seeds_by_name: dict[str, CompiledSeedView] = {seed.name: seed for seed in project.seeds} seed_fixture_names: frozenset[str] = frozenset(graph_plan.seed_fixture_names) seed_entries: list[SeedPlanEntry] = [] seed_name: str for seed_name in graph_plan.seed_names: if seed_name in seed_fixture_names: continue - seed: CompiledSeed | None = seeds_by_name.get(seed_name) + seed: CompiledSeedView | None = seeds_by_name.get(seed_name) if seed is None: raise PlannerInputError( f"Scenario '{graph_plan.name}' requires unknown seed '{seed_name}'", diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py index 92b6bae348..3776ff41f7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py @@ -15,14 +15,17 @@ ) from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, - CompiledModel, CompiledModelSqlTestPayload, CompiledObjectKey, - CompiledProject, CompiledSqlTest, CompileSqlTestCte, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + SqlTestMode, +) from sqlbuild.compiler.planner._helpers.fixtures.completion import build_relation_fixture_context from sqlbuild.compiler.planner._helpers.sql_tests.fixture_validation import ( build_validated_test_fixtures, @@ -51,7 +54,7 @@ def plan_sql_tests( *, tests: tuple[CompiledSqlTest, ...], - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, sql_analysis_enabled: bool = False, validate_fixtures: bool = False, @@ -83,7 +86,7 @@ def plan_sql_tests( render_sql=False, ) plans_by_index: dict[int, NativeSqlTestPlan] = dict(zip(plannable_indexes, plans, strict=True)) - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} results: list[SqlTestPlanResult] = [] for index, test in enumerate(tests): plan: NativeSqlTestPlan | None = plans_by_index.get(index) @@ -152,7 +155,7 @@ def _expected_column_diagnostics( def _validate_fixtures( *, tests: tuple[CompiledSqlTest, ...], - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, fixture_planning_context: RelationFixturePlanningContext, ) -> tuple[tuple[CompiledSqlTest, ...], dict[int, tuple[str, ...]]]: @@ -197,7 +200,7 @@ def _validate_fixtures( def _with_validated_fixtures( *, test: CompiledSqlTest, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, ordered_model_names: tuple[str, ...], fixture_planning_context: RelationFixturePlanningContext, @@ -273,8 +276,8 @@ def _plan_entry( *, test: CompiledSqlTest, plan: NativeSqlTestPlan, - project: CompiledProject, - models_by_name: dict[str, CompiledModel], + project: CompiledProjectView, + models_by_name: dict[str, CompiledModelView], sql_analysis_enabled: bool, ) -> SqlTestPlanEntry: mock_table_function_names: tuple[str, ...] = _prefixed_names( @@ -315,12 +318,12 @@ def _plan_entry( def _chain_function_deps( *, model_names: tuple[str, ...], - models_by_name: dict[str, CompiledModel], + models_by_name: dict[str, CompiledModelView], mocked_table_functions: frozenset[str], ) -> tuple[CompiledObjectKey, ...]: deps: dict[CompiledObjectKey, None] = {} for model_name in model_names: - model: CompiledModel | None = models_by_name.get(model_name) + model: CompiledModelView | None = models_by_name.get(model_name) if model is None: continue for dep in model.deps: @@ -335,7 +338,7 @@ def _chain_function_deps( def _direct_function_deps( - *, test: CompiledSqlTest, project: CompiledProject + *, test: CompiledSqlTest, project: CompiledProjectView ) -> tuple[CompiledObjectKey, ...]: if not isinstance(test.payload, CompiledDirectLogicSqlTestPayload): return () diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py index 59cc840666..61e38822c7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py @@ -7,7 +7,8 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics -from sqlbuild.compiler.compile.models import CompiledModel, CompiledSqlTest +from sqlbuild.compiler.compile.models import CompiledSqlTest +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics import ( render_cursor_intrinsic_bounds, ) @@ -52,7 +53,7 @@ def declares_cursor_window(*, test: CompiledSqlTest) -> bool: def render_test_cursor_intrinsics( *, sql: str, - model: CompiledModel, + model: CompiledModelView, adapter: BaseAdapter, test: CompiledSqlTest | None, ) -> str: @@ -71,11 +72,11 @@ def render_test_cursor_intrinsics( def declared_window_cursor_models( - *, test: CompiledSqlTest, chain_models: tuple[CompiledModel, ...] -) -> tuple[CompiledModel, ...]: + *, test: CompiledSqlTest, chain_models: tuple[CompiledModelView, ...] +) -> tuple[CompiledModelView, ...]: """Validate a declared window against every chain model using intrinsics and return them.""" - cursor_models: tuple[CompiledModel, ...] = tuple( + cursor_models: tuple[CompiledModelView, ...] = tuple( model for model in chain_models if uses_cursor_intrinsics(sql=model.query_sql) ) if not cursor_models: @@ -146,7 +147,7 @@ def _bound( return value -def _model_cursor(*, model: CompiledModel) -> _ModelCursor: +def _model_cursor(*, model: CompiledModelView) -> _ModelCursor: cursor_type: CursorType = CursorType( get_config_str(values=model.config.values, key="cursor_type") or CursorType.TIMESTAMP ) diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py index 5d79332892..7de7a40e3c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py @@ -6,12 +6,11 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( - CompiledProject, CompiledSqlTest, FixtureColumnInference, InferredColumn, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_completion, build_relation_fixture_context, @@ -33,7 +32,7 @@ def build_validated_test_fixtures( *, test: CompiledSqlTest, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, ordered_model_names: tuple[str, ...], mock_refs: dict[str, str], diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py index 44df9df303..9a8f915454 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py @@ -13,11 +13,10 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, - CompiledModel, CompiledModelSqlTestPayload, - CompiledProject, CompiledSqlTest, ) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.planner._helpers.sql_tests.cursor_window import ( declared_window_cursor_models, declares_cursor_window, @@ -53,7 +52,7 @@ def plan_compiled_sql_test_artifacts( *, - project: CompiledProject, + project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter, sql_analysis_enabled: bool, @@ -91,7 +90,7 @@ def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[ def plan_sql_tests_natively( *, - project: CompiledProject, + project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter, sql_analysis_enabled: bool, @@ -112,7 +111,7 @@ def plan_sql_tests_natively( def _plan_compiled_sql_tests( *, - project: CompiledProject, + project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter, sql_analysis_enabled: bool, @@ -212,11 +211,11 @@ def _mentions_cursor_intrinsic(sql: str) -> bool: def _rendered_test_overrides( - *, project: CompiledProject, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter + *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter ) -> dict[int, dict[str, str]]: """Cursor-rendered model overrides for the tests whose overrides cursor windows change.""" - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} windowed_indexes: tuple[int, ...] = tuple( index for index, test in enumerate(tests) @@ -262,7 +261,7 @@ def _record_planning_timing(*, start_ns: int, planning_ns: int, rendering_ns: in def resolve_sql_test_model_chains( - *, project: CompiledProject, tests: tuple[CompiledSqlTest, ...] + *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] ) -> tuple[tuple[str, ...], ...]: """Return each test's ordered unmocked model chain; direct-logic tests have none.""" @@ -327,7 +326,7 @@ def _unresolved_reference_error( def _function_templates( - *, project: CompiledProject, adapter: BaseAdapter + *, project: CompiledProjectView, adapter: BaseAdapter ) -> tuple[tuple[str, str, str, str, str], ...]: """Each function's name and the adapter's UDF and table-function call templates.""" @@ -357,7 +356,7 @@ def _function_templates( def _windowed_model_query_overrides( *, test: CompiledSqlTest, - models_by_name: dict[str, CompiledModel], + models_by_name: dict[str, CompiledModelView], adapter: BaseAdapter, chain: tuple[str, ...] | None, ) -> dict[str, str] | None: @@ -367,7 +366,7 @@ def _windowed_model_query_overrides( chain_names: frozenset[str] = frozenset(chain or ()) overrides: dict[str, str] = {} for model_name, sql in payload.model_query_overrides.items(): - model: CompiledModel | None = models_by_name.get(model_name) + model: CompiledModelView | None = models_by_name.get(model_name) overrides[model_name] = ( sql if model is None @@ -379,7 +378,7 @@ def _windowed_model_query_overrides( ) ) if chain is not None: - chain_models: tuple[CompiledModel, ...] = tuple( + chain_models: tuple[CompiledModelView, ...] = tuple( models_by_name[name] for name in chain if name in models_by_name ) for model in declared_window_cursor_models(test=test, chain_models=chain_models): diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py index 5ab3346d96..9c0e7e5e19 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py @@ -12,7 +12,8 @@ from sqlbuild.adapter.relations.main.list_relations_for_inspection import ( list_relations_for_inspection, ) -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import SourceEntry @@ -20,7 +21,7 @@ def get_semantic_source_columns( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, source_read_map: dict[str, SourceEntry], diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py index c7e8126b1f..ac7b1c0f54 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/snapshot.py @@ -27,16 +27,18 @@ from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY from sqlbuild.compiler.compile.main._cursor_roles import resolve_cursor_input_roles from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledRelationLocation, - CompiledSeed, - CompiledSource, CompileSqlReference, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompiledSeedView, + CompiledSourceView, +) from sqlbuild.compiler.fingerprints.constants import ( FINGERPRINT_TABLE_NAME, FUNCTION_NODE_TYPES, @@ -204,7 +206,7 @@ class _StateTableSchemas: def build_warehouse_snapshot( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, scope: PlannerScope, @@ -247,7 +249,7 @@ def build_warehouse_snapshot( def gather_warehouse_snapshot( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, execute: AdapterExecute[Any, Any], @@ -363,7 +365,7 @@ def gather_warehouse_snapshot( def gather_redirected_cursor_snapshots( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, existing_relations: dict[str, RelationInfo], @@ -401,7 +403,7 @@ def gather_redirected_cursor_snapshots( origin_cursor_column_overrides=origin_cursor_columns, ), ) - model_map: dict[str, CompiledModel] = {model.name: model for model in project.models} + model_map: dict[str, CompiledModelView] = {model.name: model for model in project.models} return { name: replace(snapshot, target_relation=model_map[name].destination.qualified_name) for name, snapshot in snapshots.items() @@ -411,7 +413,7 @@ def gather_redirected_cursor_snapshots( def _prefetch_inspection_schemas( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, database: str | None, @@ -426,7 +428,7 @@ def _prefetch_inspection_schemas( if catalog is None: return scopes: list[tuple[str | None, str]] = [(database, schema) for schema in schemas] - source: CompiledSource + source: CompiledSourceView for source in project.sources: entry: SourceEntry = source.source_entry if ( @@ -441,7 +443,7 @@ def _prefetch_inspection_schemas( def _relevant_state_keys( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None, ) -> frozenset[CompiledObjectKey] | None: """Retain state identities for the complete upstream closure of the selection.""" @@ -456,24 +458,24 @@ def _relevant_state_keys( def _resolve_database( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None, ) -> str | None: """Extract the database from the first model target that declares one.""" - model: CompiledModel + model: CompiledModelView for model in project.models: if selected_keys is not None and model.key not in selected_keys: continue if model.destination.database is not None: return model.destination.database - seed: CompiledSeed + seed: CompiledSeedView for seed in project.seeds: if selected_keys is not None and seed.key not in selected_keys: continue if seed.destination.database is not None: return seed.destination.database - function: CompiledFunction + function: CompiledFunctionView for function in project.functions: if selected_keys is not None and function.key not in selected_keys: continue @@ -484,25 +486,25 @@ def _resolve_database( def _collect_target_schemas( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None, ) -> tuple[str, ...]: """Collect distinct non-null target schemas from models, seeds, and functions.""" schemas: set[str] = set() - model: CompiledModel + model: CompiledModelView for model in project.models: if selected_keys is not None and model.key not in selected_keys: continue if model.destination.schema is not None: schemas.add(model.destination.schema) - seed: CompiledSeed + seed: CompiledSeedView for seed in project.seeds: if selected_keys is not None and seed.key not in selected_keys: continue if seed.destination.schema is not None: schemas.add(seed.destination.schema) - function: CompiledFunction + function: CompiledFunctionView for function in project.functions: if selected_keys is not None and function.key not in selected_keys: continue @@ -515,27 +517,27 @@ def _collect_target_schemas( def _build_metadata_name_filter( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None, ) -> tuple[str, ...] | None: names: set[str] = set() if selected_keys is None: - model: CompiledModel + model: CompiledModelView for model in project.models: names.add(model.destination.name) - seed: CompiledSeed + seed: CompiledSeedView for seed in project.seeds: names.add(seed.destination.name) else: selected_names: frozenset[str] = frozenset(key.name for key in selected_keys) - model_map: dict[str, CompiledModel] = {model.name: model for model in project.models} - seed_map: dict[str, CompiledSeed] = {seed.name: seed for seed in project.seeds} + model_map: dict[str, CompiledModelView] = {model.name: model for model in project.models} + seed_map: dict[str, CompiledSeedView] = {seed.name: seed for seed in project.seeds} source_map: dict[str, SourceEntry] = { source.source_entry.name: source.source_entry for source in project.sources } key: CompiledObjectKey for key in selected_keys: - selected_model: CompiledModel | None = model_map.get(key.name) + selected_model: CompiledModelView | None = model_map.get(key.name) if selected_model is not None: names.add(selected_model.destination.name) names.update( @@ -547,7 +549,7 @@ def _build_metadata_name_filter( ) ) continue - selected_seed: CompiledSeed | None = seed_map.get(key.name) + selected_seed: CompiledSeedView | None = seed_map.get(key.name) if selected_seed is not None: names.add(selected_seed.destination.name) continue @@ -571,9 +573,9 @@ def _build_metadata_name_filter( def _model_upstream_names( *, - model: CompiledModel, - model_map: dict[str, CompiledModel], - seed_map: dict[str, CompiledSeed], + model: CompiledModelView, + model_map: dict[str, CompiledModelView], + seed_map: dict[str, CompiledSeedView], selected_names: frozenset[str], ) -> frozenset[str]: names: set[str] = set() @@ -589,12 +591,12 @@ def _model_upstream_names( ): continue if reference.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModel | None = model_map.get(reference.ref_name) + upstream_model: CompiledModelView | None = model_map.get(reference.ref_name) if upstream_model is not None: names.add(upstream_model.destination.name) continue if reference.ref_kind == SqlReferenceKind.SEED: - upstream_seed: CompiledSeed | None = seed_map.get(reference.ref_name) + upstream_seed: CompiledSeedView | None = seed_map.get(reference.ref_name) if upstream_seed is not None: names.add(upstream_seed.destination.name) continue @@ -603,7 +605,7 @@ def _model_upstream_names( def _gather_relations( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, database: str | None, @@ -617,13 +619,13 @@ def _gather_relations( ) result: dict[str, RelationInfo] = {} logical_names_by_identity: dict[tuple[str | None, str | None, str], str] = {} - model: CompiledModel + model: CompiledModelView for model in project.models: logical_names_by_identity[_location_identity(model.destination)] = model.name - seed: CompiledSeed + seed: CompiledSeedView for seed in project.seeds: logical_names_by_identity[_location_identity(seed.destination)] = seed.name - function: CompiledFunction + function: CompiledFunctionView for function in project.functions: logical_names_by_identity[_location_identity(function.destination)] = function.name fingerprint_schemas: set[str] = set() @@ -667,7 +669,7 @@ def _location_identity( def _gather_columns( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, relations: dict[str, RelationInfo], @@ -782,7 +784,7 @@ def _selected_node_names( def _shares_unfiltered_fingerprints( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None, ) -> bool: """Return whether one unfiltered read can serve this selection and migration inspection.""" @@ -798,7 +800,7 @@ def _shares_unfiltered_fingerprints( def _gather_cursor_snapshots( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, execute: AdapterExecute[Any, Any], @@ -808,8 +810,8 @@ def _gather_cursor_snapshots( ) -> dict[str, ModelCursorSnapshot]: """Gather cursor MIN/MAX values for selected incremental models.""" - model_map: dict[str, CompiledModel] = {m.name: m for m in project.models} - source_map: dict[str, CompiledSource] = {s.name: s for s in project.sources} + model_map: dict[str, CompiledModelView] = {m.name: m for m in project.models} + source_map: dict[str, CompiledSourceView] = {s.name: s for s in project.sources} cursor_models: list[_CursorModelInfo] = _collect_cursor_models( project=project, @@ -855,10 +857,10 @@ def _gather_cursor_snapshots( def _collect_cursor_models( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, - model_map: dict[str, CompiledModel], - source_map: dict[str, CompiledSource], + model_map: dict[str, CompiledModelView], + source_map: dict[str, CompiledSourceView], existing_relations: dict[str, RelationInfo], inputs: _CursorGatherInputs, ) -> list[_CursorModelInfo]: @@ -874,8 +876,8 @@ def _collect_cursor_models( if effective_runtime_producer_keys is not None else None ) - seed_map: dict[str, CompiledSeed] = {seed.name: seed for seed in project.seeds} - function_map: dict[str, CompiledFunction] = { + seed_map: dict[str, CompiledSeedView] = {seed.name: seed for seed in project.seeds} + function_map: dict[str, CompiledFunctionView] = { function.name: function for function in project.functions } runtime_producer_names: frozenset[str] = _selected_runtime_producer_names( @@ -883,7 +885,7 @@ def _collect_cursor_models( selected_keys=effective_runtime_producer_keys, ) cursor_models: list[_CursorModelInfo] = [] - model: CompiledModel + model: CompiledModelView for model in project.models: if inputs.selected_keys is not None: model_key: CompiledObjectKey = CompiledObjectKey( @@ -1118,11 +1120,11 @@ def _build_cursor_queries(cursor_models: list[_CursorModelInfo]) -> list[_Physic def _cursor_input_grain( - *, ref: CompileSqlReference, model_map: dict[str, CompiledModel] + *, ref: CompileSqlReference, model_map: dict[str, CompiledModelView] ) -> str | None: if ref.ref_kind != SqlReferenceKind.REF: return None - upstream_model: CompiledModel | None = model_map.get(ref.ref_name) + upstream_model: CompiledModelView | None = model_map.get(ref.ref_name) return ( get_config_str(values=upstream_model.config.values, key="cursor_grain") if upstream_model is not None @@ -1136,7 +1138,9 @@ def _parse_optional_cursor(*, value: str | None, cursor_type: str | None) -> Cur return parse(raw=value, cursor_type=cursor_type or CursorType.TIMESTAMP) -def _has_start_override(*, model: CompiledModel, cursor_overrides: CursorOverrides | None) -> bool: +def _has_start_override( + *, model: CompiledModelView, cursor_overrides: CursorOverrides | None +) -> bool: if cursor_overrides is None: return False cursor_type: str | None = get_config_str(values=model.config.values, key="cursor_type") @@ -1583,9 +1587,9 @@ def _resolve_upstream_qualified_name( *, ref: CompileSqlReference, adapter: BaseAdapter, - model_map: dict[str, CompiledModel], - source_map: dict[str, CompiledSource], - seed_map: dict[str, CompiledSeed], + model_map: dict[str, CompiledModelView], + source_map: dict[str, CompiledSourceView], + seed_map: dict[str, CompiledSeedView], deferred_locations: dict[str, CompiledRelationLocation] | None = None, selected_names: frozenset[str] | None = None, ) -> str | None: @@ -1600,18 +1604,18 @@ def _resolve_upstream_qualified_name( ): return deferred_locations[ref.ref_name].qualified_name if ref.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModel | None = model_map.get(ref.ref_name) + upstream_model: CompiledModelView | None = model_map.get(ref.ref_name) if upstream_model is not None: return upstream_model.destination.qualified_name - upstream_seed: CompiledSeed | None = seed_map.get(ref.ref_name) + upstream_seed: CompiledSeedView | None = seed_map.get(ref.ref_name) if upstream_seed is not None: return upstream_seed.destination.qualified_name elif ref.ref_kind == SqlReferenceKind.SEED: - seed: CompiledSeed | None = seed_map.get(ref.ref_name) + seed: CompiledSeedView | None = seed_map.get(ref.ref_name) if seed is not None: return seed.destination.qualified_name elif ref.ref_kind == SqlReferenceKind.SOURCE: - source: CompiledSource | None = source_map.get(ref.ref_name) + source: CompiledSourceView | None = source_map.get(ref.ref_name) if source is not None: entry: SourceEntry = source.source_entry return render_source_relation(entry=entry, adapter=adapter) @@ -1619,7 +1623,7 @@ def _resolve_upstream_qualified_name( def _selected_runtime_producer_names( - *, project: CompiledProject, selected_keys: frozenset[CompiledObjectKey] | None + *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] | None ) -> frozenset[str]: """Return selected warehouse relations produced during this planner invocation.""" @@ -1640,19 +1644,19 @@ def _selected_runtime_producer_names( def _validate_watermark_contract_column( *, - model: CompiledModel, + model: CompiledModelView, ref: CompileSqlReference, cursor_column: str, - model_map: dict[str, CompiledModel], - source_map: dict[str, CompiledSource], - seed_map: dict[str, CompiledSeed], + model_map: dict[str, CompiledModelView], + source_map: dict[str, CompiledSourceView], + seed_map: dict[str, CompiledSeedView], ) -> None: """Validate reliable watermark column contracts before warehouse inspection.""" declared_names: tuple[str, ...] = () declared_types: dict[str, str] = {} if ref.ref_kind == SqlReferenceKind.REF: - upstream_model: CompiledModel | None = model_map.get(ref.ref_name) + upstream_model: CompiledModelView | None = model_map.get(ref.ref_name) if upstream_model is None: return upstream_cursor: str | None = get_config_str( @@ -1690,7 +1694,7 @@ def _validate_watermark_contract_column( else: return elif ref.ref_kind == SqlReferenceKind.SOURCE: - upstream_source: CompiledSource | None = source_map.get(ref.ref_name) + upstream_source: CompiledSourceView | None = source_map.get(ref.ref_name) if ( upstream_source is None or upstream_source.source_entry.contract != ContractPolicy.ENFORCED @@ -1703,7 +1707,7 @@ def _validate_watermark_contract_column( if column.type is not None } elif ref.ref_kind == SqlReferenceKind.SEED: - upstream_seed: CompiledSeed | None = seed_map.get(ref.ref_name) + upstream_seed: CompiledSeedView | None = seed_map.get(ref.ref_name) if upstream_seed is None: return declared_names = tuple(column.name for column in upstream_seed.schema_entry.columns) diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_columns.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_columns.py index 67683fc637..99e1be5814 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_columns.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_columns.py @@ -10,14 +10,14 @@ from sqlbuild.adapter.relations.main.list_relations_for_inspection import ( list_relations_for_inspection, ) -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.constants import METADATA_NAME_FILTER_LIMIT from sqlbuild.spec.contracts.models import SourceEntry def gather_source_columns( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, source_entries: tuple[SourceEntry, ...] | None = None, @@ -34,7 +34,7 @@ def gather_source_columns( def gather_source_inspection( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, source_entries: tuple[SourceEntry, ...] | None, @@ -110,7 +110,7 @@ def gather_source_inspection( def _build_source_table_name_filter( *, - project: CompiledProject, + project: CompiledProjectView, database: str | None, schemas: set[str], source_entries: tuple[SourceEntry, ...] | None = None, diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py index 870466580b..9ff46ffac2 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py @@ -7,12 +7,14 @@ from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledFunction, - CompiledModel, CompiledObjectKey, - CompiledProject, - CompiledSource, +) +from sqlbuild.compiler.compile.types import ( + CompiledAuditView, + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledSourceView, ) from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.planner.constants import ( @@ -36,7 +38,7 @@ def build_source_read_map( *, - project: CompiledProject, + project: CompiledProjectView, source_map: dict[str, SourceEntry], selected_keys: frozenset[CompiledObjectKey], project_config: ProjectConfig | None, @@ -83,7 +85,7 @@ def build_source_read_map( target_name=source_target_name, ) result: dict[str, SourceEntry] = dict(source_map) - source: CompiledSource + source: CompiledSourceView for source in project.sources: if source.source_entry.name not in selected_managed_sources: continue @@ -97,7 +99,10 @@ def build_source_read_map( def with_declared_source_reads( - *, project: CompiledProject, source_map: dict[str, SourceEntry], source_names: frozenset[str] + *, + project: CompiledProjectView, + source_map: dict[str, SourceEntry], + source_names: frozenset[str], ) -> dict[str, SourceEntry]: """Add declared project sources that no selected SQL resource reads to a source map.""" @@ -105,14 +110,14 @@ def with_declared_source_reads( if not missing: return source_map result: dict[str, SourceEntry] = dict(source_map) - source: CompiledSource + source: CompiledSourceView for source in project.sources: if source.source_entry.name in missing: result[source.source_entry.name] = source.source_entry return result -def _sql_hook_source_reads(model: CompiledModel) -> tuple[str, ...]: +def _sql_hook_source_reads(model: CompiledModelView) -> tuple[str, ...]: names: list[str] = [] for hook_key in (MODEL_PRE_HOOKS_CONFIG_KEY, MODEL_POST_HOOKS_CONFIG_KEY): value: object = model.config.values.get(hook_key) @@ -127,12 +132,12 @@ def _sql_hook_source_reads(model: CompiledModel) -> tuple[str, ...]: def _selected_managed_source_refs( *, - project: CompiledProject, + project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey], managed_source_names: frozenset[str], ) -> tuple[str, ...]: names: set[str] = set() - model: CompiledModel + model: CompiledModelView for model in project.models: if model.key not in selected_keys: continue @@ -142,7 +147,7 @@ def _selected_managed_source_refs( ) ) names.update(name for name in _sql_hook_source_reads(model) if name in managed_source_names) - function: CompiledFunction + function: CompiledFunctionView for function in project.functions: if function.key not in selected_keys: continue @@ -151,7 +156,7 @@ def _selected_managed_source_refs( sql=function.body_sql, managed_source_names=managed_source_names ) ) - audit: CompiledAudit + audit: CompiledAuditView for audit in project.audits: if audit.scope_deps and not any(dep in selected_keys for dep in audit.scope_deps): continue @@ -179,7 +184,7 @@ def _direct_managed_source_refs( def _resolve_source_target_name( *, - project: CompiledProject, + project: CompiledProjectView, project_config: ProjectConfig | None, local_config: LocalConfig | None, defer_sources_to: str | None, @@ -196,7 +201,7 @@ def _resolve_source_target_name( return active_target.defer_sources_to or project.effective_target_name -def _raw_source_entry(source: CompiledSource) -> SourceEntry: +def _raw_source_entry(source: CompiledSourceView) -> SourceEntry: source_entry: SourceEntry for source_entry in source.source_file.source_entries: if source_entry.name == source.source_entry.name: diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness.py index bad3283674..694a2469cc 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_freshness.py @@ -8,11 +8,13 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, CompiledRelationLocation, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.planner.models import ( PlannerRelationsContext, PlannerScope, @@ -29,7 +31,7 @@ def build_planner_source_freshness_result( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, scope: PlannerScope, @@ -66,7 +68,7 @@ def build_planner_source_freshness_result( ) -def _resolve_state_database(project: CompiledProject) -> str | None: +def _resolve_state_database(project: CompiledProjectView) -> str | None: model_destination: CompiledRelationLocation for model_destination in _iter_state_destinations(project=project): if model_destination.database is not None: @@ -74,7 +76,7 @@ def _resolve_state_database(project: CompiledProject) -> str | None: return None -def _collect_state_schemas(*, project: CompiledProject, scope: PlannerScope) -> tuple[str, ...]: +def _collect_state_schemas(*, project: CompiledProjectView, scope: PlannerScope) -> tuple[str, ...]: selected_model_names: frozenset[str] = frozenset( key.name for key in scope.selected_keys if key.resource_type == CompiledResourceType.MODEL ) @@ -86,13 +88,15 @@ def _collect_state_schemas(*, project: CompiledProject, scope: PlannerScope) -> if not schemas: model_name: str for model_name in selected_model_names: - model: CompiledModel | None = scope.models_by_name.get(model_name) + model: CompiledModelView | None = scope.models_by_name.get(model_name) if model is not None and model.destination.schema is not None: schemas.add(model.destination.schema) return tuple(sorted(schemas)) -def _iter_state_destinations(*, project: CompiledProject) -> tuple[CompiledRelationLocation, ...]: +def _iter_state_destinations( + *, project: CompiledProjectView +) -> tuple[CompiledRelationLocation, ...]: return ( *(model.destination for model in project.models), *(seed.destination for seed in project.seeds), diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py index 77d2a32c58..0ddfb7082e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_tables.py @@ -10,8 +10,12 @@ from sqlbuild.adapter.contract.models import RelationReadProbe from sqlbuild.adapter.contract.types import RelationReadStatus from sqlbuild.adapter.relations.main.run_bounded_inspections import run_bounded_inspections -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSource -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import ( + CompiledProjectView, + CompiledResourceType, + CompiledSourceView, +) from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line from sqlbuild.compiler.planner._helpers.resolve.sources import render_source_read_relation from sqlbuild.compiler.planner.exceptions import PlannerInputError @@ -24,7 +28,7 @@ @dataclass(frozen=True) class _MissingSourceTable: - source: CompiledSource + source: CompiledSourceView entry: SourceEntry read_by: tuple[str, ...] probe: RelationReadProbe @@ -32,7 +36,7 @@ class _MissingSourceTable: def check_selected_source_tables_exist( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, scope: PlannerScope, @@ -41,8 +45,8 @@ def check_selected_source_tables_exist( """Raise when a selected resource reads an external source table that does not exist.""" readers_by_source: dict[str, list[str]] = _selected_source_readers(scope=scope) - candidates: list[tuple[CompiledSource, SourceEntry, tuple[str, ...]]] = [] - source: CompiledSource + candidates: list[tuple[CompiledSourceView, SourceEntry, tuple[str, ...]]] = [] + source: CompiledSourceView for source in project.sources: readers: list[str] | None = readers_by_source.get(source.name) if not readers: @@ -55,7 +59,7 @@ def check_selected_source_tables_exist( candidates.append((source, entry, tuple(sorted(readers)))) if not candidates: return - unlisted: tuple[tuple[CompiledSource, SourceEntry, tuple[str, ...]], ...] = tuple( + unlisted: tuple[tuple[CompiledSourceView, SourceEntry, tuple[str, ...]], ...] = tuple( candidate for candidate in candidates if candidate[0].name not in relations.listed_source_names @@ -170,7 +174,7 @@ def _missing_source_tables_help(*, missing: tuple[_MissingSourceTable, ...]) -> ) -def _declaration_location(source: CompiledSource) -> str: +def _declaration_location(source: CompiledSourceView) -> str: path: str = source.source_file.relative_path.as_posix() line: int | None = yaml_entry_line(contents=source.source_file.contents, name=source.name) return path if line is None else f"{path}:{line}" diff --git a/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py b/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py index 933b9bc193..661972273d 100644 --- a/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py +++ b/src/sqlbuild/compiler/planner/classes/background_sql_test_planning.py @@ -7,7 +7,8 @@ from types import TracebackType from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.output.plan_output import plan_selected_sql_tests from sqlbuild.compiler.planner.models import PlannedSqlTests, SqlTestSelection @@ -18,7 +19,7 @@ class BackgroundSqlTestPlanning: def __init__( self, *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, selected_keys: frozenset[CompiledObjectKey], sql_test_selection: SqlTestSelection, @@ -67,7 +68,7 @@ def result(self) -> PlannedSqlTests: def _plan_into( *, future: Future[PlannedSqlTests], - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, selected_keys: frozenset[CompiledObjectKey], sql_test_selection: SqlTestSelection, diff --git a/src/sqlbuild/compiler/planner/main/changes/_model_changes.py b/src/sqlbuild/compiler/planner/main/changes/_model_changes.py index adced701d6..d6f30f069b 100644 --- a/src/sqlbuild/compiler/planner/main/changes/_model_changes.py +++ b/src/sqlbuild/compiler/planner/main/changes/_model_changes.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.changes.detect import ( detect_model_changes as _detect_model_changes, ) @@ -11,7 +11,7 @@ def detect_model_changes( *, - model: CompiledModel, + model: CompiledModelView, snapshot: WarehouseSnapshot, sql_analysis_enabled: bool, query_change_tracking: bool, diff --git a/src/sqlbuild/compiler/planner/main/clone/_clone.py b/src/sqlbuild/compiler/planner/main/clone/_clone.py index ed1abae730..795e3fcda5 100644 --- a/src/sqlbuild/compiler/planner/main/clone/_clone.py +++ b/src/sqlbuild/compiler/planner/main/clone/_clone.py @@ -5,7 +5,7 @@ from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.reuse.clone import ( build_clone_function_entries, build_clone_model_entries, @@ -27,14 +27,14 @@ def run_clone_planning( *, - project: CompiledProject, + project: CompiledProjectView, select: tuple[str, ...], exclude: tuple[str, ...], adapter: BaseAdapter, connection: Any, - origin_project: CompiledProject, - origin_source_project: CompiledProject, - destination_source_project: CompiledProject, + origin_project: CompiledProjectView, + origin_source_project: CompiledProjectView, + destination_source_project: CompiledProjectView, ) -> tuple[ PlanOutput, tuple[CloneSourcePlanEntry, ...], diff --git a/src/sqlbuild/compiler/planner/main/commands/_relations.py b/src/sqlbuild/compiler/planner/main/commands/_relations.py index e8fe00271d..bf8fd7840e 100644 --- a/src/sqlbuild/compiler/planner/main/commands/_relations.py +++ b/src/sqlbuild/compiler/planner/main/commands/_relations.py @@ -1,7 +1,8 @@ """Focused-command relation entrypoint.""" from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.command_planning.static import ( resolve_static_relation_context_impl, ) @@ -11,7 +12,7 @@ def resolve_static_relation_context( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, scope: PlannerScope, deferral: DeferralInputs | None = None, diff --git a/src/sqlbuild/compiler/planner/main/commands/_scope.py b/src/sqlbuild/compiler/planner/main/commands/_scope.py index e82c3a4271..e2b3c221f1 100644 --- a/src/sqlbuild/compiler/planner/main/commands/_scope.py +++ b/src/sqlbuild/compiler/planner/main/commands/_scope.py @@ -1,6 +1,6 @@ """Focused-command scope entrypoint.""" -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.command_planning.static import ( resolve_static_command_scope_impl, ) @@ -8,7 +8,7 @@ def resolve_static_command_scope( - *, project: CompiledProject, selection: PlannerSelection, auto_load_sources: bool = False + *, project: CompiledProjectView, selection: PlannerSelection, auto_load_sources: bool = False ) -> PlannerScope: """Resolve command selection through the canonical build selector phase.""" diff --git a/src/sqlbuild/compiler/planner/main/commands/audit.py b/src/sqlbuild/compiler/planner/main/commands/audit.py index 6b95d95367..79e4b27482 100644 --- a/src/sqlbuild/compiler/planner/main/commands/audit.py +++ b/src/sqlbuild/compiler/planner/main/commands/audit.py @@ -1,14 +1,14 @@ """Public focused audit-plan entrypoint.""" from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.command_planning.static import build_audit_command_plan_impl from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput def build_audit_command_plan( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, scope: PlannerScope, relations: PlannerRelationsContext, diff --git a/src/sqlbuild/compiler/planner/main/commands/relation_plan.py b/src/sqlbuild/compiler/planner/main/commands/relation_plan.py index deb661904a..d622979f2c 100644 --- a/src/sqlbuild/compiler/planner/main/commands/relation_plan.py +++ b/src/sqlbuild/compiler/planner/main/commands/relation_plan.py @@ -1,6 +1,6 @@ """Public relation-only plan entrypoint.""" -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.command_planning.static import ( build_relation_command_plan_impl, ) @@ -8,7 +8,7 @@ def build_relation_command_plan( - *, project: CompiledProject, scope: PlannerScope, relations: PlannerRelationsContext + *, project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext ) -> PlanOutput: """Project static relation data for Python checks.""" diff --git a/src/sqlbuild/compiler/planner/main/commands/seed.py b/src/sqlbuild/compiler/planner/main/commands/seed.py index 51b2de2f2b..5f6aee03b2 100644 --- a/src/sqlbuild/compiler/planner/main/commands/seed.py +++ b/src/sqlbuild/compiler/planner/main/commands/seed.py @@ -1,6 +1,6 @@ """Public focused seed-plan entrypoint.""" -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.command_planning.static import build_seed_command_plan_impl from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput @@ -8,7 +8,7 @@ def build_seed_command_plan( *, - project: CompiledProject, + project: CompiledProjectView, scope: PlannerScope, relations: PlannerRelationsContext, fingerprints: dict[str, Fingerprint] | None = None, diff --git a/src/sqlbuild/compiler/planner/main/commands/seed_state.py b/src/sqlbuild/compiler/planner/main/commands/seed_state.py index 4d0b7c8ca7..4c13face97 100644 --- a/src/sqlbuild/compiler/planner/main/commands/seed_state.py +++ b/src/sqlbuild/compiler/planner/main/commands/seed_state.py @@ -3,13 +3,13 @@ from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledSeed +from sqlbuild.compiler.compile.types import CompiledSeedView from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.command_planning.seed_state import read_seed_fingerprints def read_selected_seed_fingerprints( - *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeed, ...] + *, adapter: BaseAdapter, connection: Any, seeds: tuple[CompiledSeedView, ...] ) -> dict[str, Fingerprint]: """Read only fingerprint state required by selected direct seeds.""" diff --git a/src/sqlbuild/compiler/planner/main/commands/sql_test.py b/src/sqlbuild/compiler/planner/main/commands/sql_test.py index 20f767e92a..089f16569a 100644 --- a/src/sqlbuild/compiler/planner/main/commands/sql_test.py +++ b/src/sqlbuild/compiler/planner/main/commands/sql_test.py @@ -1,14 +1,14 @@ """Public focused SQL-test plan entrypoint.""" from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.command_planning.static import build_test_command_plan_impl from sqlbuild.compiler.planner.models import PlannerRelationsContext, PlannerScope, PlanOutput def build_test_command_plan( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, scope: PlannerScope, relations: PlannerRelationsContext, diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/typed_cursor_overrides.py b/src/sqlbuild/compiler/planner/main/cursor_window/typed_cursor_overrides.py index b33a43aec5..68493db0bd 100644 --- a/src/sqlbuild/compiler/planner/main/cursor_window/typed_cursor_overrides.py +++ b/src/sqlbuild/compiler/planner/main/cursor_window/typed_cursor_overrides.py @@ -4,7 +4,7 @@ from collections.abc import Iterable -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.resolve.cursor import ( type_cursor_overrides, ) @@ -14,7 +14,7 @@ def typed_cursor_overrides( *, cursor_overrides: CursorOverrides | None, - selected_models: Iterable[CompiledModel] = (), + selected_models: Iterable[CompiledModelView] = (), ) -> CursorOverrides | None: """Read `--start-cursor`/`--end-cursor` as the one cursor type of the selected models.""" diff --git a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py index b3cfbee3fa..65b71f266b 100644 --- a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py +++ b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py @@ -1,7 +1,8 @@ """Planner entrypoint for SQL-test artifacts planned natively from the compiled objects.""" from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.compile.models import CompiledSqlTest +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( plan_compiled_sql_test_artifacts as _plan_compiled_sql_test_artifacts, ) @@ -10,7 +11,7 @@ def plan_compiled_sql_test_artifacts( *, - project: CompiledProject, + project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter, sql_analysis_enabled: bool, diff --git a/src/sqlbuild/compiler/planner/main/execution/display_plan.py b/src/sqlbuild/compiler/planner/main/execution/display_plan.py index 7d7cce9ff5..bb59e1a16b 100644 --- a/src/sqlbuild/compiler/planner/main/execution/display_plan.py +++ b/src/sqlbuild/compiler/planner/main/execution/display_plan.py @@ -4,8 +4,8 @@ from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, ) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.graph.core import ( build_downstream_deps, build_execution_upstream_deps, @@ -23,7 +23,7 @@ def build_display_only_sqlbuild_plan( - *, project: CompiledProject, selected_model_names: tuple[str, ...], full_refresh: bool + *, project: CompiledProjectView, selected_model_names: tuple[str, ...], full_refresh: bool ) -> PlanOutput: """Build a display-only plan when warehouse planning cannot fully resolve.""" diff --git a/src/sqlbuild/compiler/planner/main/execution/execution.py b/src/sqlbuild/compiler/planner/main/execution/execution.py index 3be65d53e7..12a33f9d6a 100644 --- a/src/sqlbuild/compiler/planner/main/execution/execution.py +++ b/src/sqlbuild/compiler/planner/main/execution/execution.py @@ -9,8 +9,11 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.planner._helpers.changes.actions import resolve_model_actions from sqlbuild.compiler.planner._helpers.changes.detect import detect_changes from sqlbuild.compiler.planner._helpers.planning.buildability import ( @@ -67,7 +70,7 @@ def build_execution_plan( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, selection: PlannerSelection, @@ -209,7 +212,7 @@ def build_execution_plan( def _resolve_scopes_and_cursor_overrides( *, - project: CompiledProject, + project: CompiledProjectView, selection: PlannerSelection, policies: PlannerPolicies, overrides: PlannerOverrides, @@ -220,7 +223,7 @@ def _resolve_scopes_and_cursor_overrides( policies=policies, ) selected_scope: PlannerScope = scopes.selected_scope - selected_models: tuple[CompiledModel, ...] = tuple( + selected_models: tuple[CompiledModelView, ...] = tuple( selected_scope.models_by_name[key.name] for key in selected_scope.selected_keys if key.resource_type == CompiledResourceType.MODEL @@ -236,7 +239,7 @@ def _resolve_scopes_and_cursor_overrides( def _detect_planner_change_results( *, - project: CompiledProject, + project: CompiledProjectView, scopes: PlannerScopeResolution, snapshot: WarehouseSnapshot, identities: PlannerIdentityContext, diff --git a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py b/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py index 2353f110e2..94695b3960 100644 --- a/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py +++ b/src/sqlbuild/compiler/planner/main/execution/sql_test_model_chain.py @@ -1,13 +1,14 @@ """Public planner entrypoint for batched SQL-test model closure discovery.""" -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSqlTest +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlTest +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( resolve_sql_test_model_chains, ) def sql_test_model_chain_names_by_key( - *, project: CompiledProject, tests: tuple[CompiledSqlTest, ...] + *, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] ) -> dict[CompiledObjectKey, tuple[str, ...]]: """Return model closures resolved natively in one project batch.""" diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py index 80886d310b..0a17680710 100644 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py +++ b/src/sqlbuild/compiler/planner/main/identity/version_identity_function_hashes.py @@ -5,13 +5,13 @@ import hashlib import json -from sqlbuild.compiler.compile.models import CompiledFunction +from sqlbuild.compiler.compile.types import CompiledFunctionView from sqlbuild.compiler.planner._helpers.identity.hashing import function_definition_hash def build_function_local_hashes( *, - functions: tuple[CompiledFunction, ...], + functions: tuple[CompiledFunctionView, ...], dialect: str | None, ) -> dict[str, str]: """Derive local-only semantic hashes for functions; SQL bodies ignore layout and comments.""" diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py index ad8445d44f..9c338abf61 100644 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py +++ b/src/sqlbuild/compiler/planner/main/identity/version_identity_model_metadata.py @@ -4,8 +4,7 @@ from typing import Any -from sqlbuild.compiler.compile.models import CompiledModel -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.discovery.constants import SQL_HOOK_IDENTITY_FIELDS from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries from sqlbuild.compiler.planner._helpers.identity.model_metadata import contract_output_signature @@ -22,7 +21,7 @@ def build_model_version_identity_metadata_json( *, - model: CompiledModel, + model: CompiledModelView, function_local_hashes: dict[str, str] | None = None, hook_version_hashes: dict[str, str] | None = None, ) -> str: @@ -52,7 +51,7 @@ def build_model_version_identity_metadata_json( def _model_execution_signature( - *, model: CompiledModel, hook_version_hashes: dict[str, str] + *, model: CompiledModelView, hook_version_hashes: dict[str, str] ) -> dict[str, object]: signature: dict[str, object] = {} contract_signature: dict[str, object] | None = contract_output_signature(model=model) diff --git a/src/sqlbuild/compiler/planner/main/selection/_build_model_path_index.py b/src/sqlbuild/compiler/planner/main/selection/_build_model_path_index.py deleted file mode 100644 index 2987a45de0..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_build_model_path_index.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Model path selector index entrypoint.""" - -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject -from sqlbuild.compiler.planner._helpers.graph.selector_indexes import build_model_path_index_impl - - -def build_model_path_index(project: CompiledProject) -> dict[CompiledObjectKey, str]: - """Build a model-key-to-folder lookup from a compiled project.""" - - return build_model_path_index_impl(project) diff --git a/src/sqlbuild/compiler/planner/main/selection/_build_model_tag_index.py b/src/sqlbuild/compiler/planner/main/selection/_build_model_tag_index.py deleted file mode 100644 index 38c6378efd..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/_build_model_tag_index.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Model tag selector index entrypoint.""" - -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject -from sqlbuild.compiler.planner._helpers.graph.selector_indexes import build_model_tag_index_impl - - -def build_model_tag_index(project: CompiledProject) -> dict[str, frozenset[CompiledObjectKey]]: - """Build a tag-to-model-key lookup from a compiled project.""" - - return build_model_tag_index_impl(project) diff --git a/src/sqlbuild/compiler/planner/main/selection/_graph_selection.py b/src/sqlbuild/compiler/planner/main/selection/_graph_selection.py new file mode 100644 index 0000000000..71c954d22f --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/selection/_graph_selection.py @@ -0,0 +1,18 @@ +"""Public selector resolution over a native project graph.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.planner._helpers.graph.selectors import resolve_graph_selectors + + +def resolve_graph_selection( + *, + graph: _native.NativeProjectGraph, + select: tuple[str, ...], + exclude: tuple[str, ...], +) -> frozenset[CompiledObjectKey]: + """Resolve select/exclude strings natively, adding the functions selected models need.""" + + return resolve_graph_selectors(graph=graph, select=select, exclude=exclude) diff --git a/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py b/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py index 902ec6fc09..bc7586b9df 100644 --- a/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py +++ b/src/sqlbuild/compiler/planner/main/selection/_resolve_planner_scopes.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.planning.scopes import ( resolve_planner_scopes as _resolve_planner_scopes, ) @@ -15,7 +15,7 @@ def resolve_planner_scopes( *, - project: CompiledProject, + project: CompiledProjectView, selection: PlannerSelection, policies: PlannerPolicies, ) -> PlannerScopeResolution: diff --git a/src/sqlbuild/compiler/planner/main/selection/loader_dag.py b/src/sqlbuild/compiler/planner/main/selection/loader_dag.py index 53e1573050..5455e59d6b 100644 --- a/src/sqlbuild/compiler/planner/main/selection/loader_dag.py +++ b/src/sqlbuild/compiler/planner/main/selection/loader_dag.py @@ -2,7 +2,8 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( build_intermediate_source_map as _build_intermediate_source_map, ) @@ -10,7 +11,7 @@ def build_intermediate_source_map( - *, project: CompiledProject, selected_keys: frozenset[CompiledObjectKey] + *, project: CompiledProjectView, selected_keys: frozenset[CompiledObjectKey] ) -> dict[str, SourceEntry]: """Return synthetic source entries for selected intermediate loader nodes.""" diff --git a/src/sqlbuild/compiler/planner/main/selection/scope.py b/src/sqlbuild/compiler/planner/main/selection/scope.py index e1a9e33fa1..035d762d25 100644 --- a/src/sqlbuild/compiler/planner/main/selection/scope.py +++ b/src/sqlbuild/compiler/planner/main/selection/scope.py @@ -2,14 +2,15 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.graph.scope import build_planner_scope as _build from sqlbuild.compiler.planner.models import PlannerScope def build_planner_scope( *, - project: CompiledProject, + project: CompiledProjectView, select: tuple[str, ...] = (), exclude: tuple[str, ...] = (), auto_load_sources: bool = False, diff --git a/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py b/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py index 816d361935..6e662fdae3 100644 --- a/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py +++ b/src/sqlbuild/compiler/planner/main/selection/sqlbuild_model_selectors.py @@ -2,15 +2,15 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView def resolve_sqlbuild_model_selector_names( - *, project: CompiledProject, term: str + *, project: CompiledProjectView, term: str ) -> tuple[tuple[str, ...], str | None]: """Resolve one SQLBuild-owned model selector core to names and path translation.""" - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} if term in models_by_name: return (term,), None if term.startswith("tag:"): diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index a3959b72e0..df139f94a6 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -20,15 +20,18 @@ AuditSeverity, ) from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledRelationLocation, FunctionReturnColumn, InferredColumn, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, FunctionLanguage +from sqlbuild.compiler.compile.types import ( + AttachedAuditTargetKind, + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + FunctionLanguage, +) from sqlbuild.compiler.discovery.models import DiscoveredHookFunction, SqlTestParameterDeclaration from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.migrations.types import ( @@ -146,7 +149,7 @@ class FixtureRelationMetadata: class RelationFixturePlanningContext: """Project-wide fixture metadata shared across test and scenario planning.""" - models_by_name: dict[str, CompiledModel] + models_by_name: dict[str, CompiledModelView] relations: dict[FixtureKey, FixtureRelationMetadata] authoritative_columns: dict[FixtureKey, frozenset[str]] expected_types: dict[FixtureKey, dict[str, str]] @@ -809,7 +812,7 @@ class PlannerScope: upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] all_keys: dict[str, CompiledObjectKey] - models_by_name: dict[str, CompiledModel] + models_by_name: dict[str, CompiledModelView] selected_keys: frozenset[CompiledObjectKey] execution_order: tuple[CompiledObjectKey, ...] user_selected_keys: frozenset[CompiledObjectKey] = frozenset() @@ -837,13 +840,13 @@ class ModelPlanContext: """Relation lookups and source columns for building one model plan entry.""" model_locations: dict[str, CompiledRelationLocation] - models_by_name: dict[str, CompiledModel] + models_by_name: dict[str, CompiledModelView] seed_locations: dict[str, CompiledRelationLocation] function_locations: dict[str, CompiledRelationLocation] source_map: dict[str, SourceEntry] source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] star_exclude_keyword: str - functions_by_name: dict[str, CompiledFunction] = field(default_factory=dict) + functions_by_name: dict[str, CompiledFunctionView] = field(default_factory=dict) runtime_cursor_producer_names: frozenset[str] = frozenset() future_cursor_config: FutureCursorsConfig | None = None start_cursor_config: StartCursorsConfig | None = None @@ -1654,7 +1657,7 @@ class PlannerPolicies: class PlannerRuntime: """Resolved execution environment shared by planner phases.""" - project: CompiledProject + project: CompiledProjectView adapter: BaseAdapter connection: Any project_config: ProjectConfig | None = None @@ -1693,7 +1696,7 @@ class PlannerWarehouseState: class ModelMigrationRequest: """One destination model whose data should come from an earlier relation.""" - model: CompiledModel + model: CompiledModelView discovery: MigrationDiscovery raw_origin: str | None = None origin_location: CompiledRelationLocation | None = None diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/request.py b/src/sqlbuild/compiler/project_assembly/_helpers/request.py index 1fdf58dc0f..993e69b2db 100644 --- a/src/sqlbuild/compiler/project_assembly/_helpers/request.py +++ b/src/sqlbuild/compiler/project_assembly/_helpers/request.py @@ -3,14 +3,14 @@ from __future__ import annotations import os -import re from collections.abc import Iterable +import sqlbuild._native as _native from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text -from sqlbuild.compiler.compile.models import CompileProjectInputs, CompileSqlReference +from sqlbuild.compiler.compile.models import CompileSqlReference +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.project_assembly.constants import ( BOOLEAN_VARIABLE, - ENVIRONMENT_NAME_PATTERN, FALSE_TEXT, NATIVE_INT_MAX, NATIVE_INT_MIN, @@ -24,19 +24,17 @@ TRUE_TEXT, ) from sqlbuild.compiler.project_assembly.types import ( - AuditRow, ProjectRequestRow, ReferenceRow, SyntaxCheckRow, VariableRow, ) -from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.spec.contracts.models import DefaultsConfig, TargetConfig def project_request( *, - inputs: CompileProjectInputs, + inputs: CompileInputsView, dialect: str | None, syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], ) -> tuple[ProjectRequestRow, list[Exception | None]]: @@ -80,15 +78,7 @@ def project_request( ) for model_input, checks in zip(inputs.model_inputs, syntax_checks, strict=True) ], - [ - ( - source_input.source_entry.name, - source_input.source_entry.loader is not None, - source_input.source_entry.database, - source_input.source_entry.schema, - ) - for source_input in inputs.source_inputs - ], + [source_input.assembly_row for source_input in inputs.source_inputs], [ ( seed_input.schema_entry.name, @@ -98,23 +88,8 @@ def project_request( ) for seed_input in inputs.seed_inputs ], - [ - ( - function_node_type(return_columns=function_input.return_columns), - function_input.name, - list(function_input.tags), - _reference_rows(function_input.references), - ) - for function_input in inputs.sql_function_inputs - ], - [ - _audit_row( - references=audit_input.references, - attached_kind=audit_input.attached_target_kind, - attached_name=audit_input.attached_target_name, - ) - for audit_input in inputs.audit_inputs - ], + [function_input.assembly_row for function_input in inputs.sql_function_inputs], + [audit_input.assembly_row for audit_input in inputs.audit_inputs], ) return row, [render if isinstance(render, Exception) else None for render in renders] @@ -130,11 +105,9 @@ def _tag_rows(value: object) -> list[str]: def _environment_rows(values: Iterable[str | None]) -> list[tuple[str, str | None]]: """Python's own `os.environ` lookup of every name an `ENV:` template may read.""" - names: dict[str, None] = {} - for value in values: - if value is not None: - names.update(dict.fromkeys(re.findall(ENVIRONMENT_NAME_PATTERN, value))) - return [(name, os.environ.get(name)) for name in names] + return [ + (name, os.environ.get(name)) for name in _native.environment_template_names(list(values)) + ] def _variable_row( @@ -182,17 +155,3 @@ def _syntax_check_rows( checks: tuple[tuple[str, dict[str, str] | None], ...], ) -> list[SyntaxCheckRow]: return [(sql, list((placeholders or {}).items())) for sql, placeholders in checks] - - -def _audit_row( - *, - references: tuple[CompileSqlReference, ...], - attached_kind: object, - attached_name: str | None, -) -> AuditRow: - attached: tuple[str, str] | None = ( - (str(attached_kind), attached_name) - if attached_kind is not None and attached_name is not None - else None - ) - return (_reference_rows(references), attached) diff --git a/src/sqlbuild/compiler/project_assembly/_helpers/resources.py b/src/sqlbuild/compiler/project_assembly/_helpers/resources.py index 0447acb07b..7174dd5677 100644 --- a/src/sqlbuild/compiler/project_assembly/_helpers/resources.py +++ b/src/sqlbuild/compiler/project_assembly/_helpers/resources.py @@ -2,20 +2,12 @@ from __future__ import annotations -from dataclasses import replace from typing import cast import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompileProjectInputs, - CompileSeedInput, - CompileSourceInput, -) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.project_assembly.constants import ( COMPILE_INPUT_FAILURE, ENVIRONMENT_READ, @@ -25,16 +17,13 @@ from sqlbuild.compiler.project_assembly.models import NativeProjectResources, NativeResourceFailure from sqlbuild.compiler.project_assembly.types import ( FailureRow, - NamespaceRow, - ObjectKeyRow, ProjectResourcesRow, ) from sqlbuild.compiler.sql_analysis.constants import NATIVE_FAILURE_PREFIX -from sqlbuild.spec.contracts.models import SourceEntry def project_resources( - *, inputs: CompileProjectInputs, row: ProjectResourcesRow, held: list[Exception | None] + *, inputs: CompileInputsView, row: ProjectResourcesRow, held: list[Exception | None] ) -> NativeProjectResources: """Return the resource facts the native row describes, in input order, up to the failure.""" @@ -50,22 +39,12 @@ def project_resources( ) = row _ = [_replay_read(kind=kind, name=name) for kind, name in reads] return NativeProjectResources( - model_deps=tuple(_keys(keys) for keys in model_deps), + model_deps=tuple(model_deps), model_syntax_valid=tuple(syntax), - source_entries=tuple( - _source_entry(source_input=source_input, namespace=namespace) - for source_input, namespace in zip( - inputs.source_inputs[: len(source_namespaces)], source_namespaces, strict=True - ) - ), - seed_destinations=tuple( - _seed_destination(seed_input=seed_input, namespace=namespace) - for seed_input, namespace in zip( - inputs.seed_inputs[: len(seed_namespaces)], seed_namespaces, strict=True - ) - ), - function_deps=tuple(_keys(keys) for keys in function_deps), - audit_scope_deps=tuple(_keys(keys) for keys in audit_deps), + source_namespaces=tuple(source_namespaces), + seed_namespaces=tuple(seed_namespaces), + function_deps=tuple(function_deps), + audit_scope_deps=tuple(audit_deps), failure=None if failure is None else _failure(row=failure, held=held), ) @@ -91,32 +70,3 @@ def _replay_read(*, kind: str, name: str) -> None: COMPILE_INPUT_READS.environment_read(name) else: COMPILE_INPUT_READS.context_read(name) - - -def _keys(rows: list[ObjectKeyRow]) -> tuple[CompiledObjectKey, ...]: - return tuple( - CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name) - for kind, name in rows - ) - - -def _source_entry( - *, source_input: CompileSourceInput, namespace: tuple[str | None, str | None] | None -) -> SourceEntry: - if namespace is None: - return source_input.source_entry - return replace(source_input.source_entry, database=namespace[0], schema=namespace[1]) - - -def _seed_destination( - *, seed_input: CompileSeedInput, namespace: NamespaceRow -) -> CompiledRelationLocation: - database, schema, logical_database, logical_schema = namespace - return CompiledRelationLocation( - database=database, - schema=schema, - name=seed_input.schema_entry.name, - qualified_name=None, - logical_schema=logical_schema, - logical_database=logical_database, - ) diff --git a/src/sqlbuild/compiler/project_assembly/constants.py b/src/sqlbuild/compiler/project_assembly/constants.py index 6435d36097..78c8ec8154 100644 --- a/src/sqlbuild/compiler/project_assembly/constants.py +++ b/src/sqlbuild/compiler/project_assembly/constants.py @@ -19,4 +19,3 @@ NATIVE_INT_MIN: int = -(2**63) NATIVE_INT_MAX: int = 2**63 - 1 ENVIRONMENT_READ: str = "env" -ENVIRONMENT_NAME_PATTERN: str = r"ENV:([^\s(),'\"}]+)" diff --git a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py index 215c614cd4..a16528e4e1 100644 --- a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py +++ b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project_resources.py @@ -3,7 +3,7 @@ from __future__ import annotations import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.project_assembly._helpers.request import project_request from sqlbuild.compiler.project_assembly._helpers.resources import project_resources from sqlbuild.compiler.project_assembly.models import NativeProjectResources @@ -12,7 +12,7 @@ def assemble_native_project_resources( *, - inputs: CompileProjectInputs, + inputs: CompileInputsView, dialect: str | None, syntax_checks: tuple[tuple[tuple[str, dict[str, str] | None], ...], ...], ) -> NativeProjectResources: diff --git a/src/sqlbuild/compiler/project_assembly/models.py b/src/sqlbuild/compiler/project_assembly/models.py index 89cb67c322..b65cfeadd9 100644 --- a/src/sqlbuild/compiler/project_assembly/models.py +++ b/src/sqlbuild/compiler/project_assembly/models.py @@ -4,24 +4,20 @@ from dataclasses import dataclass -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - DynamicColumnContractProof, -) -from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild.compiler.analysis_session.types import ProofRow +from sqlbuild.compiler.project_assembly.types import NamespaceRow, ObjectKeyRow @dataclass(frozen=True, slots=True) class NativeProjectResources: """Per-resource facts in input order, and whether each model's SQL passed syntax validation.""" - model_deps: tuple[tuple[CompiledObjectKey, ...], ...] + model_deps: tuple[list[ObjectKeyRow], ...] model_syntax_valid: tuple[bool, ...] - source_entries: tuple[SourceEntry, ...] - seed_destinations: tuple[CompiledRelationLocation, ...] - function_deps: tuple[tuple[CompiledObjectKey, ...], ...] - audit_scope_deps: tuple[tuple[CompiledObjectKey, ...], ...] + source_namespaces: tuple[tuple[str | None, str | None] | None, ...] + seed_namespaces: tuple[NamespaceRow, ...] + function_deps: tuple[list[ObjectKeyRow], ...] + audit_scope_deps: tuple[list[ObjectKeyRow], ...] failure: NativeResourceFailure | None = None @@ -36,10 +32,10 @@ class NativeResourceFailure: @dataclass(frozen=True, slots=True) class NativeModelFacts: - """One model's native facts.""" + """One model's native facts; its dependencies as `(resource type, name)` rows.""" - deps: tuple[CompiledObjectKey, ...] - dynamic_contract: DynamicColumnContractProof | None + deps: list[ObjectKeyRow] + dynamic_contract: ProofRow | None syntax_valid: bool = False diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py b/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py index 965fc95118..8466c89d84 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py @@ -10,6 +10,7 @@ from sqlbuild.compiler.graph.main.sql_ref_key import sql_ref_key from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many +from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.constants import ( PATH_SELECTOR_EXPLICIT_ROOT_ERROR, @@ -19,7 +20,6 @@ from sqlbuild.compiler.planner.main.selection._build_resources import ( expand_build_resource_selection, ) -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector from sqlbuild.compiler.planner.main.selection.unit_test_selector_error import ( @@ -441,15 +441,7 @@ def _resolve_single_side( def _resolve_sql(*, raw: str, project_graph: ProjectGraph) -> frozenset[_SelectionAtom]: return frozenset( _sql_atom(key) - for key in resolve_project_selectors( - select=(raw,), - exclude=(), - all_keys=project_graph.all_keys, - upstream_deps=project_graph.upstream_deps, - downstream_deps=project_graph.downstream_deps, - tag_index=project_graph.tag_index, - path_index=project_graph.path_index, - ) + for key in select_project_graph(graph=project_graph, select=(raw,), exclude=()) ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py deleted file mode 100644 index e0399ee685..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/columns/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/columns/column_planning.py b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_planning.py deleted file mode 100644 index bcd5e6e449..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/columns/column_planning.py +++ /dev/null @@ -1,509 +0,0 @@ -"""Walk a column rename from its owner model through every downstream reader.""" - -from __future__ import annotations - -from collections import deque - -from sqlbuild.compiler.compile.models import CompiledModel -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.refactoring._helpers.columns.column_references import ( - analyze_column, - consumer_edits, - output_edits, - resource_columns, -) -from sqlbuild.compiler.refactoring._helpers.project.project_files import ( - authored_bodies, - project_sql_files, - python_string_locations, - yaml_files, -) -from sqlbuild.compiler.refactoring._helpers.renames.model_planning import ( - find_model, - needs_column_migration, - validate_identifier, -) -from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( - add_column_entry_edit, - column_config_edits, - column_entry_edits, - consumer_column_header_edits, - header_tokens, - unhandled_word_offsets, -) -from sqlbuild.compiler.refactoring._helpers.text.schema_edits import schema_column_edits -from sqlbuild.compiler.refactoring._helpers.text.sql_sites import ( - analysis_sql, - authored_offset, - authored_span, - model_body, -) -from sqlbuild.compiler.refactoring._helpers.text.text_edits import ( - manual_at, - merge_parts, - path_edits, -) -from sqlbuild.compiler.refactoring._helpers.text.yaml_edits import yaml_column_edits -from sqlbuild.compiler.refactoring.constants import ( - EXPECTED_FIXTURE_PREFIX, - FIXTURE_ROLES, - GENERIC_DIALECT, - MIGRATE_FROM_KEY, - REF_FIXTURE_PREFIX, - REF_KIND, - ROOT_SCOPE, -) -from sqlbuild.compiler.refactoring.exceptions import RefactorInputError -from sqlbuild.compiler.refactoring.models import ( - AnalysisSql, - AuthoredBody, - BodyContext, - BodyEdits, - ColumnFacts, - ColumnQuery, - ColumnRenameContext, - HeaderToken, - ManualLocation, - MigrationDeclaration, - ModelBody, - ProjectSqlFile, - RefactorParts, - RefactorProject, - RefactorRequest, - TextEdit, -) -from sqlbuild.compiler.refactoring.types import SqlFileRole -from sqlbuild.spec.contracts.models import SchemaColumn - -type _ModelContext = tuple[BodyContext | None, str, tuple[ManualLocation, ...]] - - -def column_rename_context( - *, project: RefactorProject, request: RefactorRequest -) -> ColumnRenameContext: - """Validate a column rename and collect everything its walk reads.""" - - owner: CompiledModel = find_model(project=project.graph.project, name=request.model_name) - old: str = request.column_name or "" - new: str = request.new_name - validate_identifier(name=new, noun="column") - files: tuple[ProjectSqlFile, ...] = project_sql_files(discovered=project.discovered) - context: ColumnRenameContext = ColumnRenameContext( - project=project, - owner=owner, - dialect=project.graph.project.sql_analysis_dialect or GENERIC_DIALECT, - columns=resource_columns(project=project.graph.project), - contents={item.relative_path: item.contents for item in files}, - bodies=authored_bodies(discovered=project.discovered), - yaml_files=yaml_files(discovered=project.discovered), - schema_files=tuple(item for item in files if item.role == SqlFileRole.SCHEMA), - old=old, - new=new, - cascade=request.cascade, - ) - if old.lower() not in _output_names(context=context): - raise RefactorInputError( - f"model:{owner.name} has no output column '{old}'", - code="C956", - help="column names match case-insensitively; check the spelling", - ) - if old.lower() == new.lower(): - raise RefactorInputError(f"column {old} already has that name", code="C953") - return context - - -def column_rename_parts(*, context: ColumnRenameContext) -> RefactorParts: - """Return every edit of the rename; `cascaded` lists each model whose column is renamed.""" - - parts: list[RefactorParts] = [_collision(context=context), _owner_output(context=context)] - queue: deque[CompiledModel] = deque([context.owner]) - seen: set[str] = set() - while queue: - model: CompiledModel = queue.popleft() - if model.name in seen: - continue - seen.add(model.name) - consumers: RefactorParts - cascaded: tuple[CompiledModel, ...] - consumers, cascaded = _consumers(context=context, model=model) - parts.extend( - ( - RefactorParts(cascaded=(model.name,)), - _declarations(context=context, model=model), - consumers, - _authored_body_parts(context=context, model=model), - RefactorParts( - edits=( - *yaml_column_edits( - files=context.yaml_files, - model=model.name, - old=context.old, - new=context.new, - ), - *_schema_edits(context=context, model=model), - ) - ), - ) - ) - queue.extend(cascaded) - merged: RefactorParts = merge_parts(parts=tuple(parts)) - return merge_parts(parts=(merged, _python_locations(context=context, merged=merged))) - - -def _schema_edits( - *, context: ColumnRenameContext, model: CompiledModel -) -> tuple[tuple[str, TextEdit], ...]: - edits: list[tuple[str, TextEdit]] = [] - item: ProjectSqlFile - for item in context.schema_files: - edits.extend( - path_edits( - path=item.relative_path, - edits=schema_column_edits( - contents=item.contents, - upstream=model.name, - old=context.old, - new=context.new, - ), - ) - ) - return tuple(edits) - - -def _output_names(*, context: ColumnRenameContext) -> frozenset[str]: - return frozenset( - name.lower() for name in context.columns.get((REF_KIND, context.owner.name), ()) - ) - - -def _collision(*, context: ColumnRenameContext) -> RefactorParts: - if context.new.lower() not in _output_names(context=context): - return RefactorParts() - return RefactorParts( - blocking=( - ManualLocation( - path=context.owner.relative_path.as_posix(), - line=None, - column=None, - reason=f"model:{context.owner.name} already has a column named {context.new}", - ), - ) - ) - - -def _owner_output(*, context: ColumnRenameContext) -> RefactorParts: - body: BodyContext | None - compiled_sql: str - unmapped: tuple[ManualLocation, ...] - body, compiled_sql, unmapped = _model_context(context=context, model=context.owner) - if body is None: - return RefactorParts(manual=unmapped) - facts: ColumnFacts = analyze_column( - analysis=analysis_sql(text=compiled_sql, dialect=context.dialect), - dialect=context.dialect, - columns=context.columns, - query=ColumnQuery( - column=context.old, - target_tables=frozenset(), - output_scopes=frozenset({ROOT_SCOPE}), - ), - ) - result: BodyEdits = output_edits( - facts=facts, - context=body, - scopes=frozenset({ROOT_SCOPE}), - old=context.old, - new=context.new, - ) - missing: tuple[ManualLocation, ...] = ( - () - if result.edits or result.manual - else ( - ManualLocation( - path=body.path, - line=None, - column=None, - reason=f"cannot find where model:{context.owner.name} produces {context.old}", - ), - ) - ) - return RefactorParts( - edits=path_edits(path=body.path, edits=result.edits), manual=(*result.manual, *missing) - ) - - -def _declarations(*, context: ColumnRenameContext, model: CompiledModel) -> RefactorParts: - path: str = model.relative_path.as_posix() - declared: SchemaColumn | None = next( - ( - column - for column in (model.schema_entry.columns if model.schema_entry is not None else ()) - if column.name.lower() == context.old.lower() - ), - None, - ) - if declared is not None and declared.migrate_from is not None: - return RefactorParts( - blocking=( - ManualLocation( - path=path, - line=None, - column=None, - reason=( - f"column {context.old} of model:{model.name} still declares " - "migrate_from; build it on every target and remove migrate_from before " - "renaming it again" - ), - ), - ) - ) - shared: tuple[ManualLocation, ...] = ( - ( - ManualLocation( - path=declared.location.path.as_posix(), - line=declared.location.line, - column=declared.location.column, - reason=( - f"column {context.old} of model:{model.name} is declared in a shared schema" - ), - ), - ) - if declared is not None - and declared.location is not None - and declared.location.path.as_posix() != path - else () - ) - return merge_parts( - parts=(RefactorParts(manual=shared), _header_parts(context=context, model=model)) - ) - - -def _header_parts(*, context: ColumnRenameContext, model: CompiledModel) -> RefactorParts: - path: str = model.relative_path.as_posix() - contents: str = context.contents.get(path, "") - tokens: tuple[HeaderToken, ...] = header_tokens(contents=contents) or () - migrate: bool = needs_column_migration(model=model) - entry_edits: tuple[TextEdit, ...] | None = column_entry_edits( - contents=contents, tokens=tokens, old=context.old, new=context.new, migrate=migrate - ) - added: TextEdit | None = ( - add_column_entry_edit(contents=contents, tokens=tokens, new=context.new, old=context.old) - if entry_edits is None and migrate - else None - ) - edits: tuple[TextEdit, ...] = ( - *column_config_edits(contents=contents, tokens=tokens, old=context.old, new=context.new), - *(entry_edits or ()), - *((added,) if added is not None else ()), - ) - handled: frozenset[int] = frozenset(edit.start for edit in edits) - return RefactorParts( - edits=path_edits(path=path, edits=edits), - manual=tuple( - manual_at( - path=path, - text=contents, - offset=offset, - reason=f"the MODEL header mentions {context.old} here; update it by hand", - ) - for offset in unhandled_word_offsets( - contents=contents, tokens=tokens, word=context.old, handled=handled - ) - ), - migrations=( - MigrationDeclaration( - model_name=model.name, - declaration=f"{context.new} ({MIGRATE_FROM_KEY} {context.old})", - reason="keeps the column's history in place", - ), - ) - if migrate - else (), - ) - - -def _consumers( - *, context: ColumnRenameContext, model: CompiledModel -) -> tuple[RefactorParts, tuple[CompiledModel, ...]]: - parts: list[RefactorParts] = [] - cascaded: list[CompiledModel] = [] - consumer: CompiledModel - for consumer in context.project.graph.project.models: - if not any( - dep.resource_type == CompiledResourceType.MODEL and dep.name == model.name - for dep in consumer.deps - ): - continue - consumer_parts: RefactorParts - passes_through: bool - consumer_parts, passes_through = _consumer( - context=context, upstream=model, consumer=consumer - ) - parts.append(consumer_parts) - if passes_through: - cascaded.append(consumer) - return merge_parts(parts=tuple(parts)), tuple(cascaded) - - -def _consumer( - *, context: ColumnRenameContext, upstream: CompiledModel, consumer: CompiledModel -) -> tuple[RefactorParts, bool]: - body: BodyContext | None - compiled_sql: str - unmapped: tuple[ManualLocation, ...] - body, compiled_sql, unmapped = _model_context(context=context, model=consumer) - if body is None: - return RefactorParts(manual=unmapped), False - analysis: AnalysisSql = analysis_sql(text=compiled_sql, dialect=context.dialect) - facts: ColumnFacts = analyze_column( - analysis=analysis, - dialect=context.dialect, - columns=context.columns, - query=ColumnQuery( - column=context.old, - target_tables=analysis.tables.get((REF_KIND, upstream.name), frozenset()), - ), - ) - result: BodyEdits = consumer_edits( - facts=facts, - context=body, - old=context.old, - new=context.new, - cascade_root=context.cascade, - root_stars_pass=context.cascade, - ) - header: tuple[TextEdit, ...] = consumer_column_header_edits( - contents=body.contents, upstream=upstream.name, old=context.old, new=context.new - ) - return ( - RefactorParts( - edits=path_edits(path=body.path, edits=(*result.edits, *header)), - manual=result.manual, - ), - result.passes_through, - ) - - -def _authored_body_parts(*, context: ColumnRenameContext, model: CompiledModel) -> RefactorParts: - return merge_parts( - parts=tuple( - _authored_body(context=context, model=model, body=body) - for body in context.bodies - if model.name.lower() in body.text.lower() - ) - ) - - -def _authored_body( - *, context: ColumnRenameContext, model: CompiledModel, body: AuthoredBody -) -> RefactorParts: - lowered: str = body.text.lower() - analysis: AnalysisSql = analysis_sql(text=body.text, dialect=context.dialect) - targets: frozenset[str] = analysis.tables.get((REF_KIND, model.name), frozenset()) - fixture_scopes: frozenset[str] = ( - frozenset( - name - for name in ( - f"{EXPECTED_FIXTURE_PREFIX}{model.name}", - f"{REF_FIXTURE_PREFIX}{model.name}", - ) - if name.lower() in lowered - ) - if body.role in FIXTURE_ROLES - else frozenset() - ) - if not targets and not fixture_scopes: - return RefactorParts() - facts: ColumnFacts = analyze_column( - analysis=analysis, - dialect=context.dialect, - columns=context.columns, - query=ColumnQuery( - column=context.old, - target_tables=targets, - target_ctes=frozenset( - name for name in fixture_scopes if name.startswith(REF_FIXTURE_PREFIX) - ), - output_scopes=fixture_scopes, - ), - ) - located: BodyContext = BodyContext( - path=body.path, - contents=body.contents, - map_span=lambda start, end: (body.start + start, body.start + end), - locate=lambda offset: body.start + offset, - fallback_offset=body.start, - ) - outputs: BodyEdits = output_edits( - facts=facts, context=located, scopes=fixture_scopes, old=context.old, new=context.new - ) - consumers: BodyEdits = consumer_edits( - facts=facts, - context=located, - old=context.old, - new=context.new, - cascade_root=False, - root_stars_pass=True, - ) - return RefactorParts( - edits=path_edits(path=body.path, edits=(*outputs.edits, *consumers.edits)), - manual=(*outputs.manual, *consumers.manual), - ) - - -def _python_locations(*, context: ColumnRenameContext, merged: RefactorParts) -> RefactorParts: - edited: frozenset[str] = frozenset(path for path, _ in merged.edits) - return RefactorParts( - manual=tuple( - location - for location in python_string_locations( - project_dir=context.project.project_dir, - discovered=context.project.discovered, - names=(context.old,), - reason=( - f"Python SQL may read column {context.old} of {context.owner.name}; check it " - "by hand" - ), - context=context.owner.name, - ) - if location.path not in edited - ) - ) - - -def _model_context(*, context: ColumnRenameContext, model: CompiledModel) -> _ModelContext: - path: str = model.relative_path.as_posix() - contents: str | None = context.contents.get(path) - if contents is None: - return None, "", () - body: ModelBody | None = model_body( - project=context.project.graph.project, - project_dir=context.project.project_dir, - model=model, - contents=contents, - ) - if body is None: - return ( - None, - "", - ( - ManualLocation( - path=path, - line=None, - column=None, - reason="the compiled SQL of this model cannot be mapped back to the file", - ), - ), - ) - mapped: ModelBody = body - return ( - BodyContext( - path=path, - contents=contents, - map_span=lambda start, end: authored_span(body=mapped, start=start, end=end), - locate=lambda offset: authored_offset(body=mapped, offset=offset)[0], - fallback_offset=body.body_start, - ), - body.compiled_sql, - (), - ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/columns/column_references.py b/src/sqlbuild/compiler/refactoring/_helpers/columns/column_references.py deleted file mode 100644 index ca6d6026a9..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/columns/column_references.py +++ /dev/null @@ -1,398 +0,0 @@ -"""Native column-reference facts for one SQL body, and the edits they imply.""" - -from __future__ import annotations - -import json -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.refactoring._helpers.text.text_edits import manual_at, text_edit -from sqlbuild.compiler.refactoring.constants import ( - CTE_SCOPE_PREFIX, - QUOTE_CHARACTERS, - REF_KIND, - ROOT_SCOPE, - SEED_KIND, - SOURCE_KIND, - UNKNOWN_COLUMN_TYPE, -) -from sqlbuild.compiler.refactoring.models import ( - AnalysisSql, - BodyContext, - BodyEdits, - ColumnFacts, - ColumnQuery, - ColumnReference, - ColumnSite, - ManualLocation, - OutputColumn, - TextEdit, -) -from sqlbuild.compiler.refactoring.types import EditKind, NativeColumnReferences, ResourceColumns - - -def resource_columns(*, project: CompiledProject) -> ResourceColumns: - """Return the known output columns of every model, source, and seed.""" - - columns: ResourceColumns = {} - for model in project.models: - names: list[str] = [column.name for column in model.inferred_columns or ()] - names.extend( - column.name - for column in (model.schema_entry.columns if model.schema_entry is not None else ()) - if column.name not in names - ) - columns[(REF_KIND, model.name)] = tuple(names) - for source in project.sources: - columns[(SOURCE_KIND, source.name)] = tuple( - column.name for column in source.source_entry.columns - ) - for seed in project.seeds: - columns[(SEED_KIND, seed.name)] = tuple(column.name for column in seed.schema_entry.columns) - return columns - - -def analyze_column( - *, analysis: AnalysisSql, dialect: str, columns: ResourceColumns, query: ColumnQuery -) -> ColumnFacts: - """Resolve every use of one column of the target relations in one SQL body.""" - - request: dict[str, object] = { - "sql": analysis.sql, - "dialect": dialect, - "column": query.column, - "schema": {"strict": False, "tables": _schema_tables(analysis=analysis, columns=columns)}, - "target_tables": sorted(query.target_tables), - "target_ctes": sorted(query.target_ctes), - "output_ctes": sorted(query.output_scopes), - } - payload: dict[str, Any] = json.loads( - cast(NativeColumnReferences, _native).analyze_column_references_json(json.dumps(request)) - ) - return ColumnFacts( - parsed=bool(payload.get("parsed")), - references=tuple(_reference(item) for item in payload.get("references", ())), - stars=tuple(_site(item) for item in payload.get("stars", ())), - joins=tuple(_site(item) for item in payload.get("joins", ())), - unresolved=tuple(_site(item) for item in payload.get("unresolved", ())), - outputs=tuple(_output(item) for item in payload.get("outputs", ())), - star_outputs=tuple(str(item) for item in payload.get("starOutputs", ())), - ) - - -def _schema_tables(*, analysis: AnalysisSql, columns: ResourceColumns) -> list[dict[str, object]]: - tables: list[dict[str, object]] = [] - key: tuple[str, str] - placeholders: frozenset[str] - for key, placeholders in analysis.tables.items(): - known: list[dict[str, str]] = [ - {"name": name, "type": UNKNOWN_COLUMN_TYPE} for name in columns.get(key, ()) - ] - name: str - for name in sorted(placeholders): - table: dict[str, object] = {"name": name, "columns": known} - tables.append(table) - return tables - - -def _reference(item: dict[str, Any]) -> ColumnReference: - projection: dict[str, Any] | None = item.get("projection") - return ColumnReference( - start=int(item["start"]), - end=int(item["end"]), - name_start=int(item["nameStart"]), - name_end=int(item["nameEnd"]), - scope=str(item["scope"]), - projected=projection is not None, - alias=projection.get("alias") if projection is not None else None, - alias_start=projection.get("aliasStart") if projection is not None else None, - alias_end=projection.get("aliasEnd") if projection is not None else None, - ) - - -def _site(item: dict[str, Any]) -> ColumnSite: - return ColumnSite(scope=str(item["scope"]), start=item.get("start"), end=item.get("end")) - - -def _output(item: dict[str, Any]) -> OutputColumn: - return OutputColumn( - scope=str(item["scope"]), - start=item.get("start"), - end=item.get("end"), - alias_start=item.get("aliasStart"), - alias_end=item.get("aliasEnd"), - ) - - -def consumer_edits( - *, - facts: ColumnFacts, - context: BodyContext, - old: str, - new: str, - cascade_root: bool, - root_stars_pass: bool, -) -> BodyEdits: - """Rename every reference; keep consumer output names unless the root passes it on.""" - - if not facts.parsed: - return BodyEdits( - manual=(_manual(context=context, offset=None, reason="SQL could not be analysed"),) - ) - return combine_body_edits( - results=( - *( - _reference_edits( - reference=reference, - context=context, - old=old, - new=new, - cascade_root=cascade_root, - ) - for reference in facts.references - ), - *( - _star_edits(star=star, context=context, old=old, root_stars_pass=root_stars_pass) - for star in facts.stars - ), - *( - _finding( - context=context, - site=site, - reason=f"USING or NATURAL join on {old}; rewrite it as an ON condition", - ) - for site in facts.joins - ), - *( - _finding( - context=context, - site=site, - reason=f"cannot tell whether {old} here is the renamed column; qualify it", - ) - for site in facts.unresolved - ), - ) - ) - - -def combine_body_edits(*, results: tuple[BodyEdits, ...]) -> BodyEdits: - """Concatenate the edits of several analyses of one body.""" - - edits: list[TextEdit] = [] - manual: list[ManualLocation] = [] - result: BodyEdits - for result in results: - edits.extend(result.edits) - manual.extend(result.manual) - return BodyEdits( - edits=tuple(edits), - manual=tuple(manual), - passes_through=any(result.passes_through for result in results), - ) - - -def _reference_edits( - *, reference: ColumnReference, context: BodyContext, old: str, new: str, cascade_root: bool -) -> BodyEdits: - name_span: tuple[int, int] | None = context.map_span(reference.name_start, reference.name_end) - if name_span is None: - return BodyEdits( - manual=( - _manual( - context=context, - offset=context.locate(reference.name_start), - reason=f"column {old} is referenced in SQL a macro generates; update the " - "macro call by hand", - ), - ) - ) - authored_name: str = context.contents[name_span[0] : name_span[1]] - renamed: TextEdit = _rename_span(context=context, span=name_span, new=new) - if not reference.projected: - return BodyEdits(edits=(renamed,)) - same_name_alias: bool = reference.alias is not None and reference.alias.lower() == old.lower() - if ( - reference.scope == ROOT_SCOPE - and cascade_root - and (reference.alias is None or same_name_alias) - ): - alias_span: tuple[int, int] | None = ( - context.map_span(reference.alias_start, reference.alias_end) - if same_name_alias - and reference.alias_start is not None - and reference.alias_end is not None - else None - ) - alias_edits: tuple[TextEdit, ...] = ( - (_rename_span(context=context, span=alias_span, new=new),) - if alias_span is not None - else () - ) - return BodyEdits(edits=(renamed, *alias_edits), passes_through=True) - if reference.alias is not None: - return BodyEdits(edits=(renamed,)) - end_span: tuple[int, int] | None = context.map_span(reference.start, reference.end) - if end_span is None: - return BodyEdits( - manual=( - _manual( - context=context, - offset=name_span[0], - reason=f"column {old} is projected inside a macro call", - ), - ) - ) - return BodyEdits( - edits=( - renamed, - text_edit( - text=context.contents, - start=end_span[1], - end=end_span[1], - replacement=f" AS {authored_name}", - kind=EditKind.COLUMN, - before="", - after=f"AS {authored_name}", - ), - ) - ) - - -def _star_edits( - *, star: ColumnSite, context: BodyContext, old: str, root_stars_pass: bool -) -> BodyEdits: - if star.scope.startswith(CTE_SCOPE_PREFIX): - return BodyEdits() - if star.scope == ROOT_SCOPE and root_stars_pass: - return BodyEdits(passes_through=True) - return _finding( - context=context, - site=star, - reason=( - f"SELECT * passes {old} through, so this model's output column would be renamed " - "too; rerun with --cascade or list the columns" - if star.scope == ROOT_SCOPE - else f"SELECT * in a subquery passes {old} through; list the columns" - ), - ) - - -def _finding(*, context: BodyContext, site: ColumnSite, reason: str) -> BodyEdits: - return BodyEdits( - manual=( - _manual( - context=context, offset=_site_offset(context=context, site=site), reason=reason - ), - ) - ) - - -def _rename_span(*, context: BodyContext, span: tuple[int, int], new: str) -> TextEdit: - return text_edit( - text=context.contents, - start=span[0], - end=span[1], - replacement=_respell(authored=context.contents[span[0] : span[1]], name=new), - kind=EditKind.COLUMN, - ) - - -def output_edits( - *, facts: ColumnFacts, context: BodyContext, scopes: frozenset[str], old: str, new: str -) -> BodyEdits: - """Rename the projection that names the column in the given output scopes.""" - - edits: list[TextEdit] = [] - manual: list[ManualLocation] = [] - found: set[str] = set() - output: OutputColumn - for output in facts.outputs: - found.add(output.scope) - if output.alias_start is not None and output.alias_end is not None: - alias_span: tuple[int, int] | None = context.map_span( - output.alias_start, output.alias_end - ) - if alias_span is None: - manual.append( - _manual( - context=context, - offset=context.locate(output.alias_start), - reason=f"column {old} is named inside a macro call", - ) - ) - continue - edits.append( - text_edit( - text=context.contents, - start=alias_span[0], - end=alias_span[1], - replacement=_respell( - authored=context.contents[alias_span[0] : alias_span[1]], name=new - ), - kind=EditKind.COLUMN, - ) - ) - continue - if output.start is None or output.end is None: - continue - column_span: tuple[int, int] | None = context.map_span(output.start, output.end) - if column_span is None: - manual.append( - _manual( - context=context, - offset=context.locate(output.start), - reason=f"column {old} is produced inside a macro call", - ) - ) - continue - edits.append( - text_edit( - text=context.contents, - start=column_span[1], - end=column_span[1], - replacement=f" AS {new}", - kind=EditKind.COLUMN, - before="", - after=f"AS {new}", - ) - ) - missing: frozenset[str] = frozenset( - scope for scope in scopes if _scope_role(scope) not in found - ) - scope: str - for scope in sorted(missing): - if _scope_role(scope) in facts.star_outputs: - manual.append( - _manual( - context=context, - offset=None, - reason=f"{old} comes from SELECT * in {scope}; list the columns explicitly", - ) - ) - return BodyEdits(edits=tuple(edits), manual=tuple(manual)) - - -def _scope_role(scope: str) -> str: - return scope if scope == ROOT_SCOPE else f"{CTE_SCOPE_PREFIX}{scope}" - - -def _respell(*, authored: str, name: str) -> str: - if authored and authored[0] in QUOTE_CHARACTERS: - return f"{authored[0]}{name}{QUOTE_CHARACTERS[authored[0]]}" - return name - - -def _site_offset(*, context: BodyContext, site: ColumnSite) -> int | None: - if site.start is None: - return None - return context.locate(site.start) - - -def _manual(*, context: BodyContext, offset: int | None, reason: str) -> ManualLocation: - return manual_at( - path=context.path, - text=context.contents, - offset=context.fallback_offset if offset is None else offset, - reason=reason, - ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py deleted file mode 100644 index e0399ee685..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/project/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project/project_files.py b/src/sqlbuild/compiler/refactoring/_helpers/project/project_files.py deleted file mode 100644 index 38f93ceca6..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/project/project_files.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Authored project files and SQL bodies a refactoring reads and edits.""" - -from __future__ import annotations - -import ast -from pathlib import Path - -from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditBlock, - DiscoveredProjectInputs, - DiscoveredSqlTestBlock, -) -from sqlbuild.compiler.refactoring._helpers.text.text_edits import whole_word_offsets -from sqlbuild.compiler.refactoring.constants import SQL_LIKE_PATTERN -from sqlbuild.compiler.refactoring.models import AuthoredBody, ManualLocation, ProjectSqlFile -from sqlbuild.compiler.refactoring.types import SqlFileRole - -type _AuthoredFile = tuple[SqlFileRole, Path, str, tuple[str, ...]] - - -def project_sql_files(*, discovered: DiscoveredProjectInputs) -> tuple[ProjectSqlFile, ...]: - """Return every discovered SQL file with its authored contents.""" - - files: dict[str, ProjectSqlFile] = {} - role: SqlFileRole - relative_path: Path - contents: str - for role, relative_path, contents, _ in ( - *_model_files(discovered=discovered), - *_authored_files(discovered=discovered), - ): - relative: str = relative_path.as_posix() - files.setdefault( - relative, ProjectSqlFile(relative_path=relative, contents=contents, role=role) - ) - return tuple(files[path] for path in sorted(files)) - - -def yaml_files(*, discovered: DiscoveredProjectInputs) -> tuple[ProjectSqlFile, ...]: - """Return every source and seed YAML declaration file.""" - - files: dict[str, str] = { - item.relative_path.as_posix(): item.contents - for item in (*discovered.source_files, *discovered.schema_files) - } - return tuple( - ProjectSqlFile(relative_path=path, contents=files[path], role=SqlFileRole.YAML) - for path in sorted(files) - ) - - -def authored_bodies(*, discovered: DiscoveredProjectInputs) -> tuple[AuthoredBody, ...]: - """Locate every non-model SQL body in its file, in file order.""" - - bodies: list[AuthoredBody] = [] - role: SqlFileRole - relative_path: Path - contents: str - texts: tuple[str, ...] - for role, relative_path, contents, texts in _authored_files(discovered=discovered): - cursor: int = 0 - text: str - for text in texts: - start: int = contents.find(text, cursor) if text.strip() else -1 - if start < 0: - continue - cursor = start + len(text) - bodies.append( - AuthoredBody( - path=relative_path.as_posix(), - role=role, - contents=contents, - start=start, - text=text, - ) - ) - return tuple(bodies) - - -def python_string_locations( - *, - project_dir: Path, - discovered: DiscoveredProjectInputs, - names: tuple[str, ...], - reason: str, - context: str | None = None, -) -> tuple[ManualLocation, ...]: - """Flag Python strings that name a model or column; with context, only SQL naming it too.""" - - locations: list[ManualLocation] = [] - relative_path: Path - for relative_path in _python_paths(discovered=discovered): - locations.extend( - _string_locations( - project_dir=project_dir, - relative_path=relative_path, - names=names, - reason=reason, - context=context, - ) - ) - return tuple(locations) - - -def _model_files(*, discovered: DiscoveredProjectInputs) -> tuple[_AuthoredFile, ...]: - return tuple( - (SqlFileRole.MODEL, item.relative_path, item.contents, ()) - for item in discovered.model_files - ) - - -def _authored_files(*, discovered: DiscoveredProjectInputs) -> tuple[_AuthoredFile, ...]: - files: list[_AuthoredFile] = [] - files.extend( - (SqlFileRole.TEST, item.relative_path, item.contents, _test_bodies(item.blocks)) - for item in discovered.test_files - ) - files.extend( - (SqlFileRole.SCENARIO, item.relative_path, item.contents, (item.sql_body,)) - for item in discovered.scenario_files - ) - files.extend( - (SqlFileRole.AUDIT, item.relative_path, item.contents, _audit_bodies(item.blocks)) - for item in discovered.audit_files - ) - files.extend( - (SqlFileRole.HOOK, item.relative_path, item.contents, (item.sql_body,)) - for item in discovered.sql_hook_files - ) - files.extend( - (SqlFileRole.FUNCTION, item.relative_path, item.contents, (item.body_sql,)) - for item in discovered.sql_function_files - ) - files.extend( - (SqlFileRole.SCHEMA, item.relative_path, item.contents, ()) - for item in discovered.model_schema_files - ) - return tuple(files) - - -def _test_bodies(blocks: tuple[DiscoveredSqlTestBlock, ...]) -> tuple[str, ...]: - return tuple(block.sql_body for block in blocks) - - -def _audit_bodies(blocks: tuple[DiscoveredAuditBlock, ...]) -> tuple[str, ...]: - return tuple(block.sql_body for block in blocks) - - -def _python_paths(*, discovered: DiscoveredProjectInputs) -> tuple[Path, ...]: - paths: set[Path] = set() - paths.update(item.relative_path for item in discovered.loader_functions) - paths.update(item.relative_path for item in discovered.task_functions) - paths.update(item.relative_path for item in discovered.asset_functions) - paths.update(item.relative_path for item in discovered.check_functions) - paths.update(item.relative_path for item in discovered.hook_functions) - paths.update(item.relative_path for item in discovered.audit_factories) - paths.update(item.relative_path for item in discovered.python_function_files) - paths.update(item.relative_path for item in discovered.materialization_files) - return tuple(sorted(paths)) - - -def _string_locations( - *, - project_dir: Path, - relative_path: Path, - names: tuple[str, ...], - reason: str, - context: str | None, -) -> tuple[ManualLocation, ...]: - try: - tree: ast.Module = ast.parse((project_dir / relative_path).read_text(encoding="utf-8")) - except (OSError, SyntaxError, UnicodeError): - return () - locations: list[ManualLocation] = [] - node: ast.AST - for node in ast.walk(tree): - if not isinstance(node, ast.Constant) or not isinstance(node.value, str): - continue - if _flags(value=node.value, names=names, context=context): - locations.append( - ManualLocation( - path=relative_path.as_posix(), - line=node.lineno, - column=node.col_offset + 1, - reason=reason, - ) - ) - return tuple(locations) - - -def _flags(*, value: str, names: tuple[str, ...], context: str | None) -> bool: - sql_like: bool = bool(SQL_LIKE_PATTERN.search(value)) - if context is not None and not (sql_like and whole_word_offsets(text=value, word=context)): - return False - return any( - value.strip().lower() == name.lower() - or (sql_like and bool(whole_word_offsets(text=value, word=name))) - for name in names - ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py b/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py deleted file mode 100644 index 824d494760..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/project/workspace.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Stage refactoring edits in a scratch copy, then commit them with rollback.""" - -from __future__ import annotations - -import os -import shutil -from pathlib import Path, PurePosixPath - -from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits -from sqlbuild.compiler.refactoring.constants import ( - COPIED_PROJECT_SUFFIXES, - HIDDEN_PREFIX, - IGNORED_PROJECT_DIRECTORIES, - PYCACHE_DIRECTORY, - TEXT_ENCODING, -) -from sqlbuild.compiler.refactoring.exceptions import RefactorWriteError -from sqlbuild.compiler.refactoring.models import FileChange -from sqlbuild.lint.main.write_atomically import write_atomically - - -def copy_project_inputs(*, project_dir: Path, staging_dir: Path) -> int: - """Copy every file compilation can read, skipping build output and hidden directories.""" - - copied: int = 0 - root: str - directories: list[str] - names: list[str] - for root, directories, names in os.walk(project_dir): - current: Path = Path(root) - directories[:] = sorted( - name - for name in directories - if not name.startswith(HIDDEN_PREFIX) - and not (current == project_dir and name in IGNORED_PROJECT_DIRECTORIES) - and name != PYCACHE_DIRECTORY - ) - name: str - for name in names: - source: Path = current / name - if source.suffix not in COPIED_PROJECT_SUFFIXES or name.startswith(HIDDEN_PREFIX): - continue - destination: Path = staging_dir / source.relative_to(project_dir) - destination.parent.mkdir(parents=True, exist_ok=True) - _ = shutil.copyfile(source, destination) - copied += 1 - return copied - - -def edited_contents( - *, originals: dict[str, str], changes: tuple[FileChange, ...] -) -> dict[str, str]: - """Return the new text of every changed file, keyed by its final path.""" - - return { - change.path: apply_text_edits(text=originals[change.original_path], edits=change.edits) - for change in changes - } - - -def read_originals(*, project_dir: Path, changes: tuple[FileChange, ...]) -> dict[str, str]: - """Read the current text of every file a plan changes.""" - - return { - change.original_path: (project_dir / PurePosixPath(change.original_path)).read_text( - encoding=TEXT_ENCODING - ) - for change in changes - } - - -def write_staged_changes( - *, staging_dir: Path, originals: dict[str, str], changes: tuple[FileChange, ...] -) -> tuple[Path, ...]: - """Apply a plan to the scratch copy and return the written paths.""" - - contents: dict[str, str] = edited_contents(originals=originals, changes=changes) - change: FileChange - for change in changes: - if change.moved: - (staging_dir / PurePosixPath(change.original_path)).unlink(missing_ok=True) - written: list[Path] = [] - for change in changes: - target: Path = staging_dir / PurePosixPath(change.path) - target.parent.mkdir(parents=True, exist_ok=True) - _ = target.write_text(contents[change.path], encoding=TEXT_ENCODING) - written.append(target) - prune_emptied_directories(root=staging_dir, changes=changes) - return tuple(written) - - -def commit_changes( - *, project_dir: Path, originals: dict[str, str], changes: tuple[FileChange, ...] -) -> tuple[Path, ...]: - """Write a verified plan into the project, restoring every file if any write fails.""" - - current: dict[str, str] = read_originals(project_dir=project_dir, changes=changes) - if current != originals: - changed: list[str] = sorted( - path for path in originals if current.get(path) != originals[path] - ) - raise RefactorWriteError( - f"files changed while the refactoring ran: {', '.join(changed)}", - help="nothing was written; run the command again", - ) - contents: dict[str, str] = edited_contents(originals=originals, changes=changes) - created_directories: list[Path] = [] - written: list[Path] = [] - try: - change: FileChange - for change in changes: - target: Path = project_dir / PurePosixPath(change.path) - created_directories.extend(_missing_directories(path=target.parent)) - target.parent.mkdir(parents=True, exist_ok=True) - written.append(target) - _write_file( - path=target, - contents=contents[change.path], - mode_source=project_dir / PurePosixPath(change.original_path), - ) - for change in changes: - if change.moved: - (project_dir / PurePosixPath(change.original_path)).unlink() - except BaseException: - _rollback( - project_dir=project_dir, - originals=originals, - written=written, - created_directories=created_directories, - ) - raise - prune_emptied_directories(root=project_dir, changes=changes) - return tuple(written) - - -def prune_emptied_directories(*, root: Path, changes: tuple[FileChange, ...]) -> None: - """Remove folders a move left empty, walking up to the project root.""" - - change: FileChange - for change in changes: - if not change.moved: - continue - directory: Path = (root / PurePosixPath(change.original_path)).parent - while directory != root and directory.is_dir() and not any(directory.iterdir()): - directory.rmdir() - directory = directory.parent - - -def _rollback( - *, - project_dir: Path, - originals: dict[str, str], - written: list[Path], - created_directories: list[Path], -) -> None: - original_paths: frozenset[Path] = frozenset( - project_dir / PurePosixPath(path) for path in originals - ) - path: Path - for path in written: - if path not in original_paths: - path.unlink(missing_ok=True) - relative: str - text: str - for relative, text in originals.items(): - restored: Path = project_dir / PurePosixPath(relative) - if restored.exists(): - write_atomically(path=restored, contents=text) - else: - _ = restored.write_text(text, encoding=TEXT_ENCODING) - directory: Path - for directory in reversed(created_directories): - if directory.is_dir() and not any(directory.iterdir()): - directory.rmdir() - - -def _write_file(*, path: Path, contents: str, mode_source: Path) -> None: - if not path.exists(): - _ = shutil.copy(mode_source, path) - write_atomically(path=path, contents=contents) - - -def _missing_directories(*, path: Path) -> list[Path]: - missing: list[Path] = [] - current: Path = path - while not current.exists(): - missing.append(current) - current = current.parent - return list(reversed(missing)) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py b/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py new file mode 100644 index 0000000000..df96a67817 --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/_helpers/python_strings.py @@ -0,0 +1,74 @@ +"""Python `ast` scan for strings that name a renamed model or column.""" + +from __future__ import annotations + +import ast +import re +from pathlib import Path + +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.refactoring.constants import IDENTIFIER_CHARACTERS, SQL_LIKE_PATTERN +from sqlbuild.compiler.refactoring.models import ManualLocation + + +def python_paths(*, discovered: DiscoveredProjectInputs) -> tuple[Path, ...]: + """Return the project Python files discovery found, sorted.""" + + paths: set[Path] = set() + paths.update(item.relative_path for item in discovered.loader_functions) + paths.update(item.relative_path for item in discovered.task_functions) + paths.update(item.relative_path for item in discovered.asset_functions) + paths.update(item.relative_path for item in discovered.check_functions) + paths.update(item.relative_path for item in discovered.hook_functions) + paths.update(item.relative_path for item in discovered.audit_factories) + paths.update(item.relative_path for item in discovered.python_function_files) + paths.update(item.relative_path for item in discovered.materialization_files) + return tuple(sorted(paths)) + + +def string_locations( + *, + project_dir: Path, + relative_path: Path, + names: tuple[str, ...], + reason: str, + context: str | None, +) -> tuple[ManualLocation, ...]: + """Return the string constants of one Python file that name a renamed model or column.""" + + try: + tree: ast.Module = ast.parse((project_dir / relative_path).read_text(encoding="utf-8")) + except (OSError, SyntaxError, UnicodeError): + return () + locations: list[ManualLocation] = [] + node: ast.AST + for node in ast.walk(tree): + if not isinstance(node, ast.Constant) or not isinstance(node.value, str): + continue + if _flags(value=node.value, names=names, context=context): + locations.append( + ManualLocation( + path=relative_path.as_posix(), + line=node.lineno, + column=node.col_offset + 1, + reason=reason, + ) + ) + return tuple(locations) + + +def _flags(*, value: str, names: tuple[str, ...], context: str | None) -> bool: + sql_like: bool = bool(SQL_LIKE_PATTERN.search(value)) + if context is not None and not (sql_like and _has_word(text=value, word=context)): + return False + return any( + value.strip().lower() == name.lower() or (sql_like and _has_word(text=value, word=name)) + for name in names + ) + + +def _has_word(*, text: str, word: str) -> bool: + pattern: re.Pattern[str] = re.compile( + rf"(? CompiledModel: - """Return the compiled model a target names.""" - - model: CompiledModel | None = next((item for item in project.models if item.name == name), None) - if model is None: - raise RefactorInputError( - f"no model named '{name}' in this project", - code="C951", - help="list models with sqb dag --json or sqb scope --browse models", - ) - return model - - -def validate_identifier(*, name: str, noun: str) -> None: - """Refuse a new name that is not a plain SQL identifier.""" - - if not IDENTIFIER_PATTERN.match(name): - raise RefactorInputError( - f"'{name}' is not a valid {noun} name", - code="C952", - help="use letters, digits, and underscores, starting with a letter or underscore", - ) - - -def model_target(*, project: RefactorProject, request: RefactorRequest) -> ModelTarget: - """Resolve the model, its new name, and its destination file.""" - - model: CompiledModel = find_model(project=project.graph.project, name=request.model_name) - source_path: str = model.relative_path.as_posix() - destination: str = ( - _resolve_destination( - project_dir=project.project_dir, - raw=request.destination or "", - file_name=model.relative_path.name, - ) - if request.operation == RefactorOperation.MOVE_MODEL - else PurePosixPath(source_path).with_name(f"{request.new_name}{SQL_SUFFIX}").as_posix() - ) - new: str = ( - PurePosixPath(destination).stem - if request.operation == RefactorOperation.MOVE_MODEL - else request.new_name - ) - validate_identifier(name=new, noun="model") - if new == model.name and destination == source_path: - raise RefactorInputError( - f"model:{model.name} already has that name and location", code="C953" - ) - return ModelTarget( - model=model, - request=replace(request, new_name=new, destination=destination), - source_path=source_path, - ) - - -def model_parts(*, project: RefactorProject, target: ModelTarget) -> RefactorParts: - """Return every edit, manual location, and blocker of one model rename or move.""" - - files: tuple[ProjectSqlFile, ...] = project_sql_files(discovered=project.discovered) - dialect: str = project.graph.project.sql_analysis_dialect or GENERIC_DIALECT - old: str = target.model.name - new: str = target.request.new_name - moves: tuple[tuple[str, str], ...] - move_blockers: tuple[ManualLocation, ...] - moves, move_blockers = _declaration_moves(project=project, target=target) - return RefactorParts( - edits=( - *model_reference_edits(files=files, old=old, new=new, dialect=dialect), - *yaml_model_edits(files=yaml_files(discovered=project.discovered), old=old, new=new), - ) - if new != old - else (), - manual=( - *macro_reference_locations(project=project, files=files, old=old, dialect=dialect), - *python_string_locations( - project_dir=project.project_dir, - discovered=project.discovered, - names=(old,), - reason=f"Python code names model {old}; update it by hand", - ), - ), - blocking=( - *_collisions(project=project, target=target), - *move_blockers, - *_pending_migration(files=files, target=target), - ), - moves=moves, - ) - - -def _resolve_destination(*, project_dir: Path, raw: str, file_name: str) -> str: - path: Path = Path(raw) - absolute: Path = path if path.is_absolute() else project_dir / path - if raw.endswith(PATH_SEPARATOR) or absolute.is_dir(): - absolute = absolute / file_name - resolved: Path = absolute.resolve() - if not resolved.is_relative_to(project_dir): - raise RefactorInputError(f"destination '{raw}' is outside the project", code="C955") - if resolved.suffix != SQL_SUFFIX: - raise RefactorInputError( - f"destination '{raw}' must be a .sql file or a folder ending in /", code="C955" - ) - return resolved.relative_to(project_dir).as_posix() - - -def _collisions(*, project: RefactorProject, target: ModelTarget) -> tuple[ManualLocation, ...]: - new: str = target.request.new_name - found: list[ManualLocation] = [ - ManualLocation( - path=other.relative_path.as_posix(), - line=None, - column=None, - reason=f"model:{new} already exists", - ) - for other in project.graph.project.models - if other.name.lower() == new.lower() and other.name != target.model.name - ] - if ( - target.destination != target.source_path - and (project.project_dir / PurePosixPath(target.destination)).exists() - ): - found.append( - ManualLocation( - path=target.destination, - line=None, - column=None, - reason="destination file already exists", - ) - ) - return tuple(found) - - -def _declaration_moves( - *, project: RefactorProject, target: ModelTarget -) -> tuple[tuple[tuple[str, str], ...], tuple[ManualLocation, ...]]: - """Move the declarations the model uses to where placement requires them afterwards.""" - - if PurePosixPath(target.destination).parent == PurePosixPath(target.source_path).parent: - return (), () - index: ScopeIndex = project.graph.project.scope_index - move: MovePreview | None - diagnostics: tuple[ScopeDiagnostic, ...] - move, diagnostics = preview_scope_move( - lookup=build_scope_lookup(index=index), - resource=f"{MODEL_KIND_PREFIX}{target.model.name}", - destination=target.destination, - ) - relocated: tuple[DeclarationRecord, ...] | None = ( - None - if move is None - else relocate_declarations_for_move( - index=index, - resource=ResourceIdentity(kind=ResourceKind.MODEL, name=target.model.name), - destination=target.destination, - ) - ) - if move is None or relocated is None: - return (), tuple( - ManualLocation(path=target.destination, line=None, column=None, reason=item.message) - for item in diagnostics - ) or ( - ManualLocation( - path=target.destination, - line=None, - column=None, - reason="declaration placement at the destination could not be worked out", - ), - ) - return _file_moves(project=project, index=index, relocated=relocated) - - -def _file_moves( - *, project: RefactorProject, index: ScopeIndex, relocated: tuple[DeclarationRecord, ...] -) -> tuple[tuple[tuple[str, str], ...], tuple[ManualLocation, ...]]: - destinations: dict[str, str] = {} - blocking: list[ManualLocation] = [] - record: DeclarationRecord - for record in sorted(relocated, key=lambda item: item.identity): - original: DeclarationRecord = next( - item for item in index.declarations if item.identity == record.identity - ) - label: str = _label(record.identity) - if not (project.project_dir / PurePosixPath(original.path)).is_file(): - blocking.append( - _declaration_blocker( - record=original, - reason=f"{label} must move to {record.path}, but it is not an authored " - "project file", - ) - ) - continue - planned: str | None = destinations.setdefault(original.path, record.path) - if planned != record.path: - blocking.append( - _declaration_blocker( - record=original, - reason=f"{original.path} holds declarations that must move to different " - f"folders ({planned}, {record.path}); split the file first", - ) - ) - moving: frozenset[DeclarationIdentity] = frozenset(item.identity for item in relocated) - staying: DeclarationRecord - for staying in index.declarations: - if staying.path in destinations and staying.identity not in moving: - blocking.append( - _declaration_blocker( - record=staying, - reason=f"{destinations[staying.path]} would take " - f"{_label(staying.identity)} along, which must " - f"stay in {staying.path}; split the file first", - ) - ) - new_path: str - for new_path in sorted(set(destinations.values())): - if (project.project_dir / PurePosixPath(new_path)).exists(): - blocking.append( - ManualLocation( - path=new_path, - line=None, - column=None, - reason="a declaration must move here, but the file already exists", - ) - ) - return tuple(sorted(destinations.items())), tuple(blocking) - - -def _label(identity: DeclarationIdentity) -> str: - return f"{identity.kind.value}:{identity.name}" - - -def _declaration_blocker(*, record: DeclarationRecord, reason: str) -> ManualLocation: - return ManualLocation(path=record.path, line=record.line, column=record.column, reason=reason) - - -def _pending_migration( - *, files: tuple[ProjectSqlFile, ...], target: ModelTarget -) -> tuple[ManualLocation, ...]: - if MIGRATE_FROM_KEY not in target.model.config.values: - return () - contents: str = next( - (item.contents for item in files if item.relative_path == target.source_path), "" - ) - token: HeaderToken | None = next( - ( - item - for item in header_tokens(contents=contents) or () - if item.depth == 0 and is_value(token=item, value=MIGRATE_FROM_KEY) - ), - None, - ) - return ( - manual_at( - path=target.source_path, - text=contents, - offset=token.start if token is not None else None, - reason=( - f"model:{target.model.name} still declares migrate_from; build it on every " - "target and remove migrate_from before renaming it again" - ), - ), - ) - - -def decide_model_migration( - *, before: CompiledProject, after: CompiledProject | None, old: str, new: str -) -> ModelMigrationDecision: - """Declare migrate_from whenever the relation moves, unless migrations cannot follow it.""" - - old_model: CompiledModel | None = _model(project=before, name=old) - new_model: CompiledModel | None = None if after is None else _model(project=after, name=new) - if old_model is None: - return ModelMigrationDecision(needed=False, reason="model not compiled") - materialized: str | None = get_config_str( - values=(new_model or old_model).config.values, key=MATERIALIZED_KEY - ) - if materialized not in MIGRATABLE_MATERIALIZATIONS: - return ModelMigrationDecision( - needed=False, reason=f"'{materialized}' models keep no warehouse data" - ) - if after is None or new_model is None: - return ModelMigrationDecision( - needed=True, - reason="keeps the relation's history; the destination could not be checked because " - "the edited project does not compile", - ) - if _location_key(old_model) == _location_key(new_model): - return ModelMigrationDecision(needed=False, reason="relation name is unchanged") - return _blocked_move(after=after, old_model=old_model, new_model=new_model) or ( - ModelMigrationDecision( - needed=True, reason="keeps the relation's history and its old name working" - ) - ) - - -def _blocked_move( - *, after: CompiledProject, old_model: CompiledModel, new_model: CompiledModel -) -> ModelMigrationDecision | None: - old_database: str = (old_model.destination.database or "").lower() - new_database: str = (new_model.destination.database or "").lower() - if old_database and new_database and old_database != new_database: - return ModelMigrationDecision( - needed=False, - blocked=True, - reason=( - f"moves from database {old_model.destination.database} to " - f"{new_model.destination.database}; migrations cannot cross databases" - ), - ) - old_schema: str = (old_model.destination.schema or "").lower() - project_schemas: frozenset[str] = frozenset( - (model.destination.schema or "").lower() for model in after.models - ) - if old_schema in project_schemas: - return None - return ModelMigrationDecision( - needed=False, - blocked=True, - reason=( - f"no model is left in schema {old_model.destination.schema}, so migrate_from " - f"{old_model.name} cannot find the old relation; add a schema-qualified " - "migrate_from for each target" - ), - ) - - -def needs_column_migration(*, model: CompiledModel) -> bool: - """Return whether renaming a column of this model must declare migrate_from.""" - - materialized: str | None = get_config_str(values=model.config.values, key=MATERIALIZED_KEY) - if materialized in HISTORY_MATERIALIZATIONS: - return True - return ( - materialized in MIGRATABLE_MATERIALIZATIONS - and model.config.values.get(MIGRATE_FROM_KEY) is not None - ) - - -def _model(*, project: CompiledProject, name: str) -> CompiledModel | None: - return next((model for model in project.models if model.name == name), None) - - -def _location_key(model: CompiledModel) -> tuple[str, str, str]: - return ( - (model.destination.database or "").lower(), - (model.destination.schema or "").lower(), - model.destination.name.lower(), - ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/renames/model_references.py b/src/sqlbuild/compiler/refactoring/_helpers/renames/model_references.py deleted file mode 100644 index 9064c7bd39..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/renames/model_references.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Every authored reference to a model by name, and the ones a rename cannot rewrite.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.refactoring._helpers.text.header_edits import model_name_header_edits -from sqlbuild.compiler.refactoring._helpers.text.schema_edits import schema_model_name_edits -from sqlbuild.compiler.refactoring._helpers.text.sql_sites import ( - authored_offset, - model_body, - resource_sites, -) -from sqlbuild.compiler.refactoring._helpers.text.text_edits import ( - identifier_sites, - manual_at, - path_edits, - text_edit, -) -from sqlbuild.compiler.refactoring.constants import ( - EXPECTED_FIXTURE_PREFIX, - FIXTURE_ROLES, - REF_FIXTURE_PREFIX, - REF_KIND, -) -from sqlbuild.compiler.refactoring.models import ( - ManualLocation, - ModelBody, - ProjectSqlFile, - RefactorProject, - ResourceSite, - TextEdit, -) -from sqlbuild.compiler.refactoring.types import EditKind, SqlFileRole - - -def model_reference_edits( - *, files: tuple[ProjectSqlFile, ...], old: str, new: str, dialect: str -) -> tuple[tuple[str, TextEdit], ...]: - """Rewrite `__ref`, fixture CTE names, and header references to a renamed model.""" - - edits: list[tuple[str, TextEdit]] = [] - item: ProjectSqlFile - for item in files: - edits.extend( - path_edits( - path=item.relative_path, - edits=_ref_edits(item=item, old=old, new=new, dialect=dialect), - ) - ) - if item.role in FIXTURE_ROLES: - edits.extend( - path_edits( - path=item.relative_path, edits=_fixture_edits(item=item, old=old, new=new) - ) - ) - if item.role == SqlFileRole.MODEL: - edits.extend( - path_edits( - path=item.relative_path, - edits=model_name_header_edits(contents=item.contents, old=old, new=new), - ) - ) - if item.role == SqlFileRole.SCHEMA: - edits.extend( - path_edits( - path=item.relative_path, - edits=schema_model_name_edits(contents=item.contents, old=old, new=new), - ) - ) - return tuple(edits) - - -def macro_reference_locations( - *, project: RefactorProject, files: tuple[ProjectSqlFile, ...], old: str, dialect: str -) -> tuple[ManualLocation, ...]: - """Locate `__ref` calls to the model that macros produce, which no edit can reach.""" - - compiled: CompiledProject = project.graph.project - contents: dict[str, str] = {item.relative_path: item.contents for item in files} - locations: list[ManualLocation] = [] - model: CompiledModel - for model in compiled.models: - text: str | None = contents.get(model.relative_path.as_posix()) - if text is None or not _depends_on(model=model, name=old): - continue - locations.extend( - _generated_references( - compiled=compiled, project=project, model=model, text=text, old=old, dialect=dialect - ) - ) - return tuple(locations) - - -def _ref_edits(*, item: ProjectSqlFile, old: str, new: str, dialect: str) -> tuple[TextEdit, ...]: - return tuple( - text_edit( - text=item.contents, - start=site.name_start, - end=site.name_end, - replacement=new, - kind=EditKind.REFERENCE, - before=item.contents[site.start : site.end], - after=( - item.contents[site.start : site.name_start] - + new - + item.contents[site.name_end : site.end] - ), - ) - for site in _ref_sites(text=item.contents, name=old, dialect=dialect) - ) - - -def _fixture_edits(*, item: ProjectSqlFile, old: str, new: str) -> tuple[TextEdit, ...]: - fixture_names: dict[str, str] = { - f"{REF_FIXTURE_PREFIX}{old}": f"{REF_FIXTURE_PREFIX}{new}", - f"{EXPECTED_FIXTURE_PREFIX}{old}": f"{EXPECTED_FIXTURE_PREFIX}{new}", - } - return tuple( - text_edit( - text=item.contents, - start=start, - end=end, - replacement=fixture_names[name], - kind=EditKind.FIXTURE, - ) - for start, end, name in identifier_sites(text=item.contents, names=frozenset(fixture_names)) - ) - - -def _generated_references( - *, - compiled: CompiledProject, - project: RefactorProject, - model: CompiledModel, - text: str, - old: str, - dialect: str, -) -> tuple[ManualLocation, ...]: - relative: str = model.relative_path.as_posix() - compiled_sites: tuple[ResourceSite, ...] = _ref_sites( - text=model.query_sql, name=old, dialect=dialect - ) - authored_sites: tuple[ResourceSite, ...] = _ref_sites( - text=model.authored_query_sql, name=old, dialect=dialect - ) - if len(compiled_sites) <= len(authored_sites): - return () - body: ModelBody | None = model_body( - project=compiled, project_dir=project.project_dir, model=model, contents=text - ) - if body is None: - return ( - manual_at( - path=relative, - text=text, - offset=None, - reason=f"reference to {old} produced by a macro", - ), - ) - offsets: tuple[tuple[int, bool], ...] = tuple( - authored_offset(body=body, offset=site.start) for site in compiled_sites - ) - return tuple( - manual_at( - path=relative, - text=text, - offset=offset, - reason=f"reference to {old} produced by a macro; pass the model to the macro as an " - "argument", - ) - for offset, generated in offsets - if generated - ) - - -def _ref_sites(*, text: str, name: str, dialect: str) -> tuple[ResourceSite, ...]: - return tuple( - site - for site in resource_sites(text=text, dialect=dialect) - if site.kind == REF_KIND and site.name == name - ) - - -def _depends_on(*, model: CompiledModel, name: str) -> bool: - return any( - dep.resource_type == CompiledResourceType.MODEL and dep.name == name for dep in model.deps - ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py b/src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py deleted file mode 100644 index e0399ee685..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/text/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Refactoring helpers.""" diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/header_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/header_edits.py deleted file mode 100644 index c7e3fbc272..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/text/header_edits.py +++ /dev/null @@ -1,449 +0,0 @@ -"""Token-exact MODEL header edits for renamed models and columns.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery.main._model_header_spans import get_model_header_spans -from sqlbuild.compiler.discovery.models import ModelHeaderSpans -from sqlbuild.compiler.refactoring._helpers.text.sql_sites import embedded_ref_spans -from sqlbuild.compiler.refactoring._helpers.text.text_edits import text_edit, whole_word_offsets -from sqlbuild.compiler.refactoring.constants import ( - COLUMN_VALUED_CONFIG_KEYS, - COLUMNS_KEY, - CURSOR_INPUTS_KEY, - HEADER_CLOSERS, - HEADER_DESCRIPTION_KEY, - HEADER_INDENT, - HEADER_KEY_AND_VALUE_TOKENS, - HEADER_OPEN_PAREN, - HEADER_OPENERS, - HEADER_SEPARATOR, - IDENTIFIER_PATTERN, - NATIVE_HEADER_TOKEN_KINDS, - PARENTHESIZED_EMPTY_TOKENS, - REF_FUNCTION, - RELATIONSHIPS_AUDIT, - RELATIONSHIPS_FIELD_KEY, - RELATIONSHIPS_TO_KEY, -) -from sqlbuild.compiler.refactoring.models import ( - HeaderEntry, - HeaderToken, - RelationshipTokens, - TextEdit, -) -from sqlbuild.compiler.refactoring.types import EditKind, HeaderTokenKind -from sqlbuild.lint.main.quoted_value_end import quoted_value_end - -_VALUE_KINDS: frozenset[HeaderTokenKind] = frozenset({HeaderTokenKind.WORD, HeaderTokenKind.STRING}) - - -def header_tokens(*, contents: str) -> tuple[HeaderToken, ...] | None: - """Tokenize the MODEL header body in file offsets, or None without a header.""" - - spans: ModelHeaderSpans = get_model_header_spans(contents=contents) - if spans.body is None: - return None - return span_tokens(contents=contents, span=spans.body) - - -def span_tokens(*, contents: str, span: tuple[int, int]) -> tuple[HeaderToken, ...]: - """Tokenize one declaration header body in file offsets.""" - - body_start: int = span[0] - body: str = contents[span[0] : span[1]] - tokens: list[HeaderToken] = [] - depth: int = 0 - raw_kind: int - value: str - position: int - for raw_kind, value, position in _native.tokenize_model_header(body): - kind: HeaderTokenKind = NATIVE_HEADER_TOKEN_KINDS[raw_kind] - if kind == HeaderTokenKind.END: - continue - start: int = body_start + position - end: int = _token_end(contents=contents, kind=kind, value=value, start=start) - if kind == HeaderTokenKind.SYMBOL and value in HEADER_CLOSERS: - depth -= 1 - tokens.append(HeaderToken(kind=kind, value=value, start=start, end=end, depth=depth)) - if kind == HeaderTokenKind.SYMBOL and value in HEADER_OPENERS: - depth += 1 - return tuple(tokens) - - -def header_entries(*, tokens: tuple[HeaderToken, ...]) -> tuple[HeaderEntry, ...]: - """Split header tokens into top-level entries keyed by their first word.""" - - return tuple( - HeaderEntry(key=first.value, tokens=(first, *rest)) - for first, rest in _nested_entries(tokens=tokens, depth=0) - if first.kind == HeaderTokenKind.WORD - ) - - -def rename_value_token( - *, contents: str, token: HeaderToken, new_value: str, kind: EditKind -) -> TextEdit: - """Replace one word or string value, keeping string quoting.""" - - replacement: str = ( - new_value - if token.kind == HeaderTokenKind.WORD - else f"{contents[token.start]}{new_value}{contents[token.start]}" - ) - return text_edit( - text=contents, - start=token.start, - end=token.end, - replacement=replacement, - kind=kind, - ) - - -def is_value(*, token: HeaderToken, value: str) -> bool: - """Return whether a word or string token spells a name, ignoring case.""" - - return token.kind in _VALUE_KINDS and token.value.lower() == value.lower() - - -def cursor_input_tokens( - *, tokens: tuple[HeaderToken, ...] -) -> tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...]: - """Return (input name token, value tokens) for every cursor_inputs entry.""" - - entry: HeaderEntry - for entry in header_entries(tokens=tokens): - if entry.key != CURSOR_INPUTS_KEY: - continue - return _nested_entries(tokens=entry.tokens[1:], depth=1) - return () - - -def relationship_tokens(*, tokens: tuple[HeaderToken, ...]) -> tuple[RelationshipTokens, ...]: - """Return the target and field value tokens of every relationships audit.""" - - found: list[RelationshipTokens] = [] - index: int - token: HeaderToken - for index, token in enumerate(tokens): - if not ( - token.kind == HeaderTokenKind.WORD - and token.value == RELATIONSHIPS_AUDIT - and index + 1 < len(tokens) - and tokens[index + 1].value == HEADER_OPEN_PAREN - ): - continue - depth: int = tokens[index + 1].depth + 1 - values: dict[str, int] = {} - cursor: int = index + 2 - while cursor < len(tokens) and tokens[cursor].depth >= depth: - candidate: HeaderToken = tokens[cursor] - if ( - candidate.depth == depth - and candidate.kind == HeaderTokenKind.WORD - and candidate.value in {RELATIONSHIPS_TO_KEY, RELATIONSHIPS_FIELD_KEY} - and cursor + 1 < len(tokens) - ): - values[candidate.value] = cursor + 1 - cursor += 1 - found.append( - _relationship( - tokens=tokens, - to_index=values.get(RELATIONSHIPS_TO_KEY), - field_index=values.get(RELATIONSHIPS_FIELD_KEY), - ) - ) - return tuple(found) - - -def column_config_edits( - *, contents: str, tokens: tuple[HeaderToken, ...], old: str, new: str -) -> tuple[TextEdit, ...]: - """Rename a column in column-valued config such as cursor or unique_key.""" - - edits: list[TextEdit] = [] - entry: HeaderEntry - for entry in header_entries(tokens=tokens): - if entry.key not in COLUMN_VALUED_CONFIG_KEYS: - continue - token: HeaderToken - for token in entry.tokens[1:]: - if is_value(token=token, value=old): - edits.append( - rename_value_token( - contents=contents, token=token, new_value=new, kind=EditKind.HEADER - ) - ) - return tuple(edits) - - -def column_entry_edits( - *, contents: str, tokens: tuple[HeaderToken, ...], old: str, new: str, migrate: bool -) -> tuple[TextEdit, ...] | None: - """Rename a `columns` entry and add `migrate_from` if needed; None without an entry.""" - - entry: HeaderEntry - for entry in header_entries(tokens=tokens): - if entry.key != COLUMNS_KEY: - continue - nested: tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...] = _nested_entries( - tokens=entry.tokens[1:], depth=1 - ) - name_token: HeaderToken - values: tuple[HeaderToken, ...] - for name_token, values in nested: - if not is_value(token=name_token, value=old): - continue - edits: list[TextEdit] = [ - rename_value_token( - contents=contents, token=name_token, new_value=new, kind=EditKind.HEADER - ) - ] - if migrate: - edits.append( - _column_migration_edit( - contents=contents, name=name_token, values=values, old=old - ) - ) - return tuple(edits) - return None - - -def add_column_entry_edit( - *, contents: str, tokens: tuple[HeaderToken, ...], new: str, old: str -) -> TextEdit | None: - """Declare `new (migrate_from old)` in the columns block, creating it when absent.""" - - declaration: str = f"{new} (migrate_from {old})" - entry: HeaderEntry - for entry in header_entries(tokens=tokens): - if entry.key != COLUMNS_KEY or len(entry.tokens) < HEADER_KEY_AND_VALUE_TOKENS: - continue - opener: HeaderToken = entry.tokens[1] - indentation: str = _line_indent(contents=contents, offset=entry.tokens[0].start) - return text_edit( - text=contents, - start=opener.end, - end=opener.end, - replacement=f"\n{indentation}{HEADER_INDENT}{declaration},", - kind=EditKind.MIGRATION, - before="", - after=declaration, - ) - return insert_header_entry_edit( - contents=contents, entry=f"columns ({declaration})", display=declaration - ) - - -def insert_header_entry_edit(*, contents: str, entry: str, display: str) -> TextEdit | None: - """Insert one top-level entry as the first MODEL header line.""" - - spans: ModelHeaderSpans = get_model_header_spans(contents=contents) - if spans.body is None: - return None - start: int = spans.body[0] - body: str = contents[spans.body[0] : spans.body[1]] - if body.startswith("\n"): - line_end: int = body.find("\n", 1) - first_line: str = body[1:] if line_end < 0 else body[1:line_end] - indentation: str = first_line[: len(first_line) - len(first_line.lstrip())] or HEADER_INDENT - replacement: str = f"\n{indentation}{entry}," - elif body.strip(): - replacement = f"{entry}, " - else: - replacement = f"\n{HEADER_INDENT}{entry},\n" - return text_edit( - text=contents, - start=start, - end=start, - replacement=replacement, - kind=EditKind.MIGRATION, - before="", - after=display, - ) - - -def unhandled_word_offsets( - *, contents: str, tokens: tuple[HeaderToken, ...], word: str, handled: frozenset[int] -) -> tuple[int, ...]: - """Return header offsets that still mention a word outside descriptions.""" - - offsets: list[int] = [] - previous: HeaderToken | None = None - token: HeaderToken - for token in tokens: - described: bool = previous is not None and previous.value == HEADER_DESCRIPTION_KEY - previous = token - if token.kind == HeaderTokenKind.SYMBOL or described or token.start in handled: - continue - text: str = contents[token.start : token.end] - offsets.extend(token.start + offset for offset in whole_word_offsets(text=text, word=word)) - return tuple(offsets) - - -def model_name_header_edits(*, contents: str, old: str, new: str) -> tuple[TextEdit, ...]: - """Rename a model in cursor_inputs keys, bare relationships targets, and quoted SQL.""" - - tokens: tuple[HeaderToken, ...] | None = header_tokens(contents=contents) - if tokens is None: - return () - return model_name_token_edits(contents=contents, tokens=tokens, old=old, new=new) - - -def model_name_token_edits( - *, contents: str, tokens: tuple[HeaderToken, ...], old: str, new: str -) -> tuple[TextEdit, ...]: - """Rename a model in the given header tokens.""" - - names: list[HeaderToken] = [ - name for name, _ in cursor_input_tokens(tokens=tokens) if is_value(token=name, value=old) - ] - names.extend( - relationship.target - for relationship in relationship_tokens(tokens=tokens) - if relationship.target is not None - and not relationship.called - and is_value(token=relationship.target, value=old) - ) - edits: list[TextEdit] = [ - rename_value_token(contents=contents, token=name, new_value=new, kind=EditKind.HEADER) - for name in names - ] - token: HeaderToken - for token in tokens: - if token.kind == HeaderTokenKind.STRING: - edits.extend( - text_edit( - text=contents, - start=token.start + start, - end=token.start + end, - replacement=new, - kind=EditKind.REFERENCE, - ) - for start, end in embedded_ref_spans( - text=contents[token.start : token.end], name=old - ) - ) - return tuple(edits) - - -def consumer_column_header_edits( - *, contents: str, upstream: str, old: str, new: str -) -> tuple[TextEdit, ...]: - """Rename an upstream column in cursor_inputs values and relationships `field` values.""" - - tokens: tuple[HeaderToken, ...] | None = header_tokens(contents=contents) - if tokens is None: - return () - return column_token_edits(contents=contents, tokens=tokens, upstream=upstream, old=old, new=new) - - -def column_token_edits( - *, contents: str, tokens: tuple[HeaderToken, ...], upstream: str, old: str, new: str -) -> tuple[TextEdit, ...]: - """Rename an upstream column in the given header tokens.""" - - values: list[HeaderToken] = [] - name: HeaderToken - inputs: tuple[HeaderToken, ...] - for name, inputs in cursor_input_tokens(tokens=tokens): - if is_value(token=name, value=upstream): - values.extend(value for value in inputs if is_value(token=value, value=old)) - relationship: RelationshipTokens - for relationship in relationship_tokens(tokens=tokens): - if ( - relationship.target is not None - and relationship.field is not None - and is_value(token=relationship.target, value=upstream) - and is_value(token=relationship.field, value=old) - ): - values.append(relationship.field) - return tuple( - rename_value_token(contents=contents, token=value, new_value=new, kind=EditKind.HEADER) - for value in values - ) - - -def quoted_name(name: str) -> str: - """Return a header value spelling for a name.""" - - return name if IDENTIFIER_PATTERN.match(name) else f'"{name}"' - - -def _column_migration_edit( - *, contents: str, name: HeaderToken, values: tuple[HeaderToken, ...], old: str -) -> TextEdit: - declaration: str = f"migrate_from {old}" - if values and values[0].value == HEADER_OPEN_PAREN: - opener: HeaderToken = values[0] - has_metadata: bool = len(values) > PARENTHESIZED_EMPTY_TOKENS - return text_edit( - text=contents, - start=opener.end, - end=opener.end, - replacement=f"{declaration}, " if has_metadata else declaration, - kind=EditKind.MIGRATION, - before="", - after=declaration, - ) - return text_edit( - text=contents, - start=name.end, - end=name.end, - replacement=f" ({declaration})", - kind=EditKind.MIGRATION, - before="", - after=declaration, - ) - - -def _relationship( - *, tokens: tuple[HeaderToken, ...], to_index: int | None, field_index: int | None -) -> RelationshipTokens: - field: HeaderToken | None = tokens[field_index] if field_index is not None else None - if to_index is None: - return RelationshipTokens(target=None, field=field, called=False) - called: bool = ( - tokens[to_index].value == REF_FUNCTION - and to_index + 2 < len(tokens) - and tokens[to_index + 1].value == HEADER_OPEN_PAREN - and tokens[to_index + 2].kind == HeaderTokenKind.STRING - ) - return RelationshipTokens( - target=tokens[to_index + 2] if called else tokens[to_index], field=field, called=called - ) - - -def _nested_entries( - *, tokens: tuple[HeaderToken, ...], depth: int -) -> tuple[tuple[HeaderToken, tuple[HeaderToken, ...]], ...]: - """Split the contents of one bracketed value into (first token, rest) entries.""" - - groups: list[list[HeaderToken]] = [[]] - token: HeaderToken - for token in tokens: - if token.depth < depth: - continue - if token.depth == depth and _is_separator(token): - groups.append([]) - continue - groups[-1].append(token) - return tuple( - (group[0], tuple(group[1:])) for group in groups if group and group[0].kind in _VALUE_KINDS - ) - - -def _is_separator(token: HeaderToken) -> bool: - return token.kind == HeaderTokenKind.SYMBOL and token.value == HEADER_SEPARATOR - - -def _token_end(*, contents: str, kind: HeaderTokenKind, value: str, start: int) -> int: - if kind != HeaderTokenKind.STRING: - return start + len(value) - return quoted_value_end(text=contents, start=start) - - -def _line_indent(*, contents: str, offset: int) -> str: - line_start: int = contents.rfind("\n", 0, offset) + 1 - prefix: str = contents[line_start:offset] - return prefix if not prefix.strip() else HEADER_INDENT diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py deleted file mode 100644 index 98cda55188..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/text/schema_edits.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Model and column references in reusable SCHEMA declaration headers.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( - column_token_edits, - model_name_token_edits, - span_tokens, -) -from sqlbuild.compiler.refactoring.constants import SCHEMA_STATEMENT_PATTERN -from sqlbuild.compiler.refactoring.models import HeaderToken, TextEdit - - -def schema_model_name_edits(*, contents: str, old: str, new: str) -> tuple[TextEdit, ...]: - """Rename a model in every SCHEMA header of a declaration file.""" - - return _each_schema( - contents=contents, - edit=lambda tokens: model_name_token_edits( - contents=contents, tokens=tokens, old=old, new=new - ), - ) - - -def schema_column_edits( - *, contents: str, upstream: str, old: str, new: str -) -> tuple[TextEdit, ...]: - """Rename relationships `field` values pointing at a renamed column of a model.""" - - return _each_schema( - contents=contents, - edit=lambda tokens: column_token_edits( - contents=contents, tokens=tokens, upstream=upstream, old=old, new=new - ), - ) - - -def _each_schema( - *, contents: str, edit: Callable[[tuple[HeaderToken, ...]], tuple[TextEdit, ...]] -) -> tuple[TextEdit, ...]: - edits: list[TextEdit] = [] - for match in SCHEMA_STATEMENT_PATTERN.finditer(contents): - edits.extend( - edit(span_tokens(contents=contents, span=(match.start("header"), match.end("header")))) - ) - return tuple(edits) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/sql_sites.py b/src/sqlbuild/compiler/refactoring/_helpers/text/sql_sites.py deleted file mode 100644 index 4f209e46a3..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/text/sql_sites.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Reference call sites, offset-preserving analysis SQL, and compiled-to-authored offsets.""" - -from __future__ import annotations - -import re -from pathlib import Path - -from sqlbuild.compiler.compile.main.map_expanded_offset import map_expanded_offset -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, - CompiledSqlExpansion, - ExpansionSpan, - MappedOffset, -) -from sqlbuild.compiler.refactoring.constants import ( - EMBEDDED_REF_PATTERN, - GENERIC_PLACEHOLDER_KIND, - PLACEHOLDER_BASE, - PLACEHOLDER_PAD, - RESOURCE_CALL_PATTERN, -) -from sqlbuild.compiler.refactoring.models import AnalysisSql, ModelBody, ResourceSite -from sqlbuild.lint.main.scan_interpolation_sites import scan_interpolation_sites -from sqlbuild.lint.models import InterpolationSite - - -def resource_sites(*, text: str, dialect: str) -> tuple[ResourceSite, ...]: - """Return resource calls outside comments and strings, with their name spans.""" - - sites: list[ResourceSite] = [] - site: InterpolationSite - for site in scan_interpolation_sites(body=text, dialect=dialect): - match: re.Match[str] | None = RESOURCE_CALL_PATTERN.match(site.original_text) - if match is None: - continue - sites.append( - ResourceSite( - kind=match.group("kind"), - name=match.group("name"), - start=site.original_start, - end=site.original_end, - name_start=site.original_start + match.start("name"), - name_end=site.original_start + match.end("name"), - ) - ) - return tuple(sites) - - -def embedded_ref_spans(*, text: str, name: str) -> tuple[tuple[int, int], ...]: - """Return the name spans of `__ref` calls to a model inside a quoted string's raw text.""" - - return tuple( - (match.start("name"), match.end("name")) - for match in EMBEDDED_REF_PATTERN.finditer(text) - if match.group("name") == name - ) - - -def analysis_sql(*, text: str, dialect: str) -> AnalysisSql: - """Replace every interpolation site with an identifier of the same length.""" - - pieces: list[str] = [] - tables: dict[tuple[str, str], set[str]] = {} - copied_to: int = 0 - index: int - site: InterpolationSite - for index, site in enumerate(scan_interpolation_sites(body=text, dialect=dialect)): - match: re.Match[str] | None = RESOURCE_CALL_PATTERN.match(site.original_text) - kind: str = match.group("kind")[0] if match is not None else GENERIC_PLACEHOLDER_KIND - placeholder: str = _placeholder( - kind=kind, index=index, length=site.original_end - site.original_start - ) - if match is not None: - tables.setdefault((match.group("kind"), match.group("name")), set()).add(placeholder) - pieces.extend((text[copied_to : site.original_start], placeholder)) - copied_to = site.original_end - pieces.append(text[copied_to:]) - return AnalysisSql( - sql="".join(pieces), - tables={key: frozenset(value) for key, value in tables.items()}, - ) - - -def model_body( - *, project: CompiledProject, project_dir: Path, model: CompiledModel, contents: str -) -> ModelBody | None: - """Return the mappable compiled body of a model, or None when it cannot be mapped.""" - - body_start: int = contents.rfind(model.authored_query_sql) if model.authored_query_sql else -1 - if body_start < 0: - return None - expansion: CompiledSqlExpansion | None = _expansion( - project=project, project_dir=project_dir, model=model - ) - passes: tuple[tuple[ExpansionSpan, ...], ...] = () - if expansion is not None: - if expansion.expanded_sql != model.query_sql: - return None - passes = expansion.passes - elif model.query_sql != model.authored_query_sql: - return None - return ModelBody( - relative_path=model.relative_path.as_posix(), - contents=contents, - body_start=body_start, - compiled_sql=model.query_sql, - passes=passes, - ) - - -def authored_offset(*, body: ModelBody, offset: int) -> tuple[int, bool]: - """Return the file offset of a compiled offset and whether a macro generated it.""" - - mapped: MappedOffset = map_expanded_offset(offset=offset, passes=body.passes) - return body.body_start + mapped.offset, mapped.generated - - -def authored_span(*, body: ModelBody, start: int, end: int) -> tuple[int, int] | None: - """Return the file span of compiled text written by the author, or None if generated.""" - - mapped_start: int - start_generated: bool - mapped_start, start_generated = authored_offset(body=body, offset=start) - if start_generated: - return None - if end <= start: - return mapped_start, mapped_start - mapped_last: int - last_generated: bool - mapped_last, last_generated = authored_offset(body=body, offset=end - 1) - if last_generated or mapped_last - mapped_start != end - 1 - start: - return None - return mapped_start, mapped_last + 1 - - -def _expansion( - *, project: CompiledProject, project_dir: Path, model: CompiledModel -) -> CompiledSqlExpansion | None: - target: Path = (project_dir / model.relative_path).resolve() - path: Path - expansion: CompiledSqlExpansion - for path, expansion in project.sql_expansions.items(): - if path.resolve() == target: - return expansion - return None - - -def _placeholder(*, kind: str, index: int, length: int) -> str: - base: str = PLACEHOLDER_BASE.format(kind=kind, index=index) - if len(base) > length: - return PLACEHOLDER_PAD * (length - 1) + kind - return base + PLACEHOLDER_PAD * (length - len(base)) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/text_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/text_edits.py deleted file mode 100644 index a82cd19f21..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/text/text_edits.py +++ /dev/null @@ -1,193 +0,0 @@ -"""Offsets, positions, and edit application for authored text.""" - -from __future__ import annotations - -import re -from functools import cache - -from sqlbuild.compiler.refactoring.constants import IDENTIFIER_CHARACTERS, NON_CODE_PATTERN -from sqlbuild.compiler.refactoring.exceptions import RefactorEditError -from sqlbuild.compiler.refactoring.models import ( - FileChange, - ManualLocation, - MigrationDeclaration, - RefactorParts, - RefactorPlan, - RefactorRequest, - TextEdit, -) -from sqlbuild.compiler.refactoring.types import EditKind - - -def line_column(*, text: str, offset: int) -> tuple[int, int]: - """Return the 1-based line and column of an offset.""" - - line_start: int = text.rfind("\n", 0, offset) + 1 - return text.count("\n", 0, offset) + 1, offset - line_start + 1 - - -def text_edit( - *, - text: str, - start: int, - end: int, - replacement: str, - kind: EditKind, - before: str | None = None, - after: str | None = None, -) -> TextEdit: - """Build one edit with its position and display text.""" - - line: int - column: int - line, column = line_column(text=text, offset=start) - return TextEdit( - start=start, - end=end, - replacement=replacement, - kind=kind, - line=line, - column=column, - before=text[start:end] if before is None else before, - after=replacement.strip() if after is None else after, - ) - - -def manual_at(*, path: str, text: str, offset: int | None, reason: str) -> ManualLocation: - """Build a manual location at an optional offset.""" - - if offset is None: - return ManualLocation(path=path, line=None, column=None, reason=reason) - line: int - column: int - line, column = line_column(text=text, offset=offset) - return ManualLocation(path=path, line=line, column=column, reason=reason) - - -def apply_text_edits(*, text: str, edits: tuple[TextEdit, ...]) -> str: - """Apply non-overlapping edits back to front; identical duplicates apply once.""" - - unique: list[TextEdit] = sorted( - {(edit.start, edit.end, edit.replacement): edit for edit in edits}.values(), - key=lambda edit: (edit.start, edit.end), - reverse=True, - ) - result: str = text - previous_start: int = len(text) + 1 - edit: TextEdit - for edit in unique: - if edit.end > previous_start: - raise RefactorEditError(f"overlapping refactoring edits at offset {edit.start}") - result = result[: edit.start] + edit.replacement + result[edit.end :] - previous_start = edit.start - return result - - -def merge_parts(*, parts: tuple[RefactorParts, ...]) -> RefactorParts: - """Concatenate planning parts in order.""" - - edits: list[tuple[str, TextEdit]] = [] - manual: list[ManualLocation] = [] - blocking: list[ManualLocation] = [] - migrations: list[MigrationDeclaration] = [] - cascaded: list[str] = [] - part: RefactorParts - for part in parts: - edits.extend(part.edits) - manual.extend(part.manual) - blocking.extend(part.blocking) - migrations.extend(part.migrations) - cascaded.extend(part.cascaded) - return RefactorParts( - edits=tuple(edits), - manual=tuple(manual), - blocking=tuple(blocking), - migrations=tuple(migrations), - cascaded=tuple(cascaded), - ) - - -def path_edits(*, path: str, edits: tuple[TextEdit, ...]) -> tuple[tuple[str, TextEdit], ...]: - """Attach a file path to each edit.""" - - return tuple((path, edit) for edit in edits) - - -def file_changes( - *, edits: tuple[tuple[str, TextEdit], ...], moves: dict[str, str] -) -> tuple[FileChange, ...]: - """Group edits by file, move files, and order the result by final path.""" - - grouped: dict[str, list[TextEdit]] = {path: [] for path in moves} - path: str - edit: TextEdit - for path, edit in edits: - grouped.setdefault(path, []).append(edit) - return tuple( - sorted( - ( - FileChange(path=moves.get(path, path), original_path=path, edits=tuple(items)) - for path, items in grouped.items() - ), - key=lambda change: change.path, - ) - ) - - -def build_plan( - *, - request: RefactorRequest, - parts: RefactorParts, - moves: dict[str, str], - renamed_columns: tuple[tuple[str, str, str], ...] = (), - help: str | None = None, -) -> RefactorPlan: - """Assemble a plan from merged parts; help applies only when manual locations exist.""" - - return RefactorPlan( - request=request, - changes=file_changes(edits=parts.edits, moves=moves), - manual=parts.manual, - blocking=parts.blocking, - migrations=parts.migrations, - renamed_columns=renamed_columns, - help=help if parts.manual else None, - ) - - -def identifier_sites(*, text: str, names: frozenset[str]) -> tuple[tuple[int, int, str], ...]: - """Return unquoted identifier tokens matching names, outside comments and strings.""" - - if not names: - return () - lowered: dict[str, str] = {name.lower(): name for name in names} - sites: list[tuple[int, int, str]] = [] - match: re.Match[str] - for match in _identifier_pattern(frozenset(lowered)).finditer(text): - token: str | None = match.group("identifier") - if token is not None: - sites.append( - (match.start("identifier"), match.end("identifier"), lowered[token.lower()]) - ) - return tuple(sites) - - -def whole_word_offsets(*, text: str, word: str) -> tuple[int, ...]: - """Return the offsets of a case-insensitive whole identifier word anywhere in text.""" - - pattern: re.Pattern[str] = re.compile( - rf"(? re.Pattern[str]: - ordered: list[str] = sorted(names, key=lambda name: (-len(name), name)) - alternatives: str = "|".join(re.escape(name) for name in ordered) - return re.compile( - NON_CODE_PATTERN - + rf"|(?{alternatives})" - + rf"(?![{IDENTIFIER_CHARACTERS}])", - re.I, - ) diff --git a/src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py b/src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py deleted file mode 100644 index aec4de9fa9..0000000000 --- a/src/sqlbuild/compiler/refactoring/_helpers/text/yaml_edits.py +++ /dev/null @@ -1,171 +0,0 @@ -"""Model and column references in source and seed YAML declarations.""" - -from __future__ import annotations - -import yaml -from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode - -from sqlbuild.compiler.refactoring._helpers.text.sql_sites import embedded_ref_spans -from sqlbuild.compiler.refactoring._helpers.text.text_edits import path_edits, text_edit -from sqlbuild.compiler.refactoring.constants import ( - RELATIONSHIPS_AUDIT, - RELATIONSHIPS_FIELD_KEY, - RELATIONSHIPS_TO_KEY, -) -from sqlbuild.compiler.refactoring.models import ProjectSqlFile, TextEdit, YamlRelationship -from sqlbuild.compiler.refactoring.types import EditKind - - -def yaml_model_edits( - *, files: tuple[ProjectSqlFile, ...], old: str, new: str -) -> tuple[tuple[str, TextEdit], ...]: - """Rename `__ref` calls in YAML strings and bare relationships targets naming a model.""" - - edits: list[tuple[str, TextEdit]] = [] - item: ProjectSqlFile - for item in files: - root: Node | None = _compose(item.contents) - if root is None: - continue - found: list[TextEdit] = [] - scalar: ScalarNode - for scalar in _scalars(root): - found.extend( - text_edit( - text=item.contents, - start=_start(scalar) + start, - end=_start(scalar) + end, - replacement=new, - kind=EditKind.REFERENCE, - ) - for start, end in embedded_ref_spans( - text=_raw(contents=item.contents, scalar=scalar), name=old - ) - ) - found.extend( - _rename_scalar( - contents=item.contents, - scalar=relationship.target, - new=new, - kind=EditKind.REFERENCE, - ) - for relationship in _relationships(root) - if relationship.target is not None and relationship.target.value == old - ) - edits.extend(path_edits(path=item.relative_path, edits=tuple(found))) - return tuple(edits) - - -def yaml_column_edits( - *, files: tuple[ProjectSqlFile, ...], model: str, old: str, new: str -) -> tuple[tuple[str, TextEdit], ...]: - """Rename relationships `field` values that point at a renamed column of a model.""" - - edits: list[tuple[str, TextEdit]] = [] - item: ProjectSqlFile - for item in files: - root: Node | None = _compose(item.contents) - if root is None: - continue - edits.extend( - path_edits( - path=item.relative_path, - edits=tuple( - _rename_scalar( - contents=item.contents, - scalar=relationship.field, - new=new, - kind=EditKind.COLUMN, - ) - for relationship in _relationships(root) - if relationship.field is not None - and relationship.field.value.lower() == old.lower() - and _targets(contents=item.contents, relationship=relationship, model=model) - ), - ) - ) - return tuple(edits) - - -def _compose(contents: str) -> Node | None: - try: - return yaml.compose(contents, Loader=yaml.SafeLoader) - except yaml.YAMLError: - return None - - -def _scalars(node: Node) -> tuple[ScalarNode, ...]: - if isinstance(node, ScalarNode): - return (node,) - children: list[Node] = [] - if isinstance(node, SequenceNode): - children.extend(node.value) - if isinstance(node, MappingNode): - children.extend(value for _, value in node.value) - found: list[ScalarNode] = [] - child: Node - for child in children: - found.extend(_scalars(child)) - return tuple(found) - - -def _relationships(node: Node) -> tuple[YamlRelationship, ...]: - found: list[YamlRelationship] = [] - if isinstance(node, MappingNode): - key: Node - value: Node - for key, value in node.value: - if ( - isinstance(key, ScalarNode) - and key.value == RELATIONSHIPS_AUDIT - and isinstance(value, MappingNode) - ): - found.append(_relationship(value)) - found.extend(_relationships(value)) - if isinstance(node, SequenceNode): - child: Node - for child in node.value: - found.extend(_relationships(child)) - return tuple(found) - - -def _relationship(node: MappingNode) -> YamlRelationship: - values: dict[str, ScalarNode] = { - key.value: value - for key, value in node.value - if isinstance(key, ScalarNode) and isinstance(value, ScalarNode) - } - return YamlRelationship( - target=values.get(RELATIONSHIPS_TO_KEY), field=values.get(RELATIONSHIPS_FIELD_KEY) - ) - - -def _targets(*, contents: str, relationship: YamlRelationship, model: str) -> bool: - target: ScalarNode | None = relationship.target - if target is None: - return False - return target.value == model or bool( - embedded_ref_spans(text=_raw(contents=contents, scalar=target), name=model) - ) - - -def _rename_scalar(*, contents: str, scalar: ScalarNode, new: str, kind: EditKind) -> TextEdit: - raw: str = _raw(contents=contents, scalar=scalar) - offset: int = max(raw.lower().find(scalar.value.lower()), 0) - start: int = _start(scalar) + offset - return text_edit( - text=contents, - start=start, - end=start + len(scalar.value), - replacement=new, - kind=kind, - ) - - -def _raw(*, contents: str, scalar: ScalarNode) -> str: - end: int = scalar.end_mark.index if scalar.end_mark is not None else _start(scalar) - return contents[_start(scalar) : end] - - -def _start(scalar: ScalarNode) -> int: - return scalar.start_mark.index if scalar.start_mark is not None else 0 diff --git a/src/sqlbuild/compiler/refactoring/constants.py b/src/sqlbuild/compiler/refactoring/constants.py index 1ef0eabe47..4343bf87af 100644 --- a/src/sqlbuild/compiler/refactoring/constants.py +++ b/src/sqlbuild/compiler/refactoring/constants.py @@ -4,108 +4,17 @@ import re -from sqlbuild.compiler.discovery.constants import STATEMENT_HEADER_BODY_PATTERN from sqlbuild.compiler.refactoring.types import ( - HeaderTokenKind, RefactorOperation, - SqlFileRole, ) MODEL_KIND_PREFIX: str = "model:" -SQL_SUFFIX: str = ".sql" -PATH_SEPARATOR: str = "/" -IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -REF_FUNCTION: str = "__ref" -REF_CALL_PATTERN: re.Pattern[str] = re.compile( - r"^__ref\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" -) -RESOURCE_CALL_PATTERN: re.Pattern[str] = re.compile( - r"^__(?Pref|source|seed)\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" -) -EMBEDDED_REF_PATTERN: re.Pattern[str] = re.compile( - r"""__ref\s*\(\s*\\?['"](?P[^'"\\]+)\\?['"]\s*\)""" -) -REF_FIXTURE_PREFIX: str = "__ref__" -EXPECTED_FIXTURE_PREFIX: str = "__expected__" -ASSERT_FIXTURE_PREFIX: str = "__assert__" -NON_CODE_PATTERN: str = ( - r"--[^\n]*(?:\n|\Z)" - r"|/\*[\s\S]*?(?:\*/|\Z)" - r"|'(?:''|[^'])*(?:'|\Z)" - r'|"(?:""|[^"])*(?:"|\Z)' -) IDENTIFIER_CHARACTERS: str = r"A-Za-z0-9_$" -PLACEHOLDER_PAD: str = "_" -GENERIC_PLACEHOLDER_KIND: str = "x" -ROOT_SCOPE: str = "root" -CTE_SCOPE_PREFIX: str = "cte:" MIGRATE_FROM_KEY: str = "migrate_from" -MATERIALIZED_KEY: str = "materialized" -CURSOR_INPUTS_KEY: str = "cursor_inputs" -COLUMNS_KEY: str = "columns" -RELATIONSHIPS_AUDIT: str = "relationships" -RELATIONSHIPS_TO_KEY: str = "to" -RELATIONSHIPS_FIELD_KEY: str = "field" -COLUMN_VALUED_CONFIG_KEYS: frozenset[str] = frozenset( - { - "cursor", - "unique_key", - "updated_at", - "check_columns", - "merge_exclude_columns", - "partition_by", - "cluster_by", - "observed_at", - "row_diff_exclude_columns", - } -) -HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset({"incremental", "snapshot"}) -MIGRATABLE_MATERIALIZATIONS: frozenset[str] = frozenset( - {"table", "view", "incremental", "snapshot"} -) -COPIED_PROJECT_SUFFIXES: frozenset[str] = frozenset( - {".sql", ".py", ".yml", ".yaml", ".toml", ".csv", ".json"} -) -IGNORED_PROJECT_DIRECTORIES: frozenset[str] = frozenset( - {"target", "logs", "venv", "node_modules", "__pycache__"} -) SQL_LIKE_PATTERN: re.Pattern[str] = re.compile(r"\b(select|from|join|insert|update)\b", re.I) OPERATION_TITLES: dict[RefactorOperation, str] = { RefactorOperation.RENAME_MODEL: "Rename model", RefactorOperation.MOVE_MODEL: "Move model", RefactorOperation.RENAME_COLUMN: "Rename column", } -PLACEHOLDER_BASE: str = "_q{kind}{index}" -QUOTE_CHARACTERS: dict[str, str] = {'"': '"', "`": "`", "[": "]"} -COLUMN_TARGET_SEPARATOR: str = "." -HEADER_OPENERS: frozenset[str] = frozenset({"(", "[", "{"}) -HEADER_CLOSERS: frozenset[str] = frozenset({")", "]", "}"}) -HEADER_SEPARATOR: str = "," -HEADER_OPEN_PAREN: str = "(" -HEADER_DESCRIPTION_KEY: str = "description" -HEADER_INDENT: str = " " -PARENTHESIZED_EMPTY_TOKENS: int = 2 -REF_KIND: str = "ref" -FIXTURE_ROLES: frozenset[SqlFileRole] = frozenset({SqlFileRole.TEST, SqlFileRole.SCENARIO}) -HIDDEN_PREFIX: str = "." -TEXT_ENCODING: str = "utf-8" GENERIC_DIALECT: str = "generic" -NATIVE_HEADER_TOKEN_KINDS: dict[int, HeaderTokenKind] = { - 0: HeaderTokenKind.END, - 1: HeaderTokenKind.WORD, - 2: HeaderTokenKind.STRING, - 3: HeaderTokenKind.SYMBOL, -} -HEADER_KEY_AND_VALUE_TOKENS: int = 2 -PYCACHE_DIRECTORY: str = "__pycache__" -SOURCE_KIND: str = "source" -SEED_KIND: str = "seed" -UNKNOWN_COLUMN_TYPE: str = "UNKNOWN" -MODEL_MANUAL_HELP: str = ( - "Pass the model into the macro as an argument, or edit the listed locations, then run the " - "command again." -) -COLUMN_MANUAL_HELP: str = "Edit the listed locations, then run the command again." -SCHEMA_STATEMENT_PATTERN: re.Pattern[str] = re.compile( - r"\bSCHEMA\s*\(" + STATEMENT_HEADER_BODY_PATTERN + r"\)\s*;" -) diff --git a/src/sqlbuild/compiler/refactoring/exceptions.py b/src/sqlbuild/compiler/refactoring/exceptions.py index 4226649f3f..4769fbb810 100644 --- a/src/sqlbuild/compiler/refactoring/exceptions.py +++ b/src/sqlbuild/compiler/refactoring/exceptions.py @@ -25,3 +25,11 @@ class RefactorWriteError(RefactorInputError): """Project files changed between planning and writing.""" code: str = "C958" + + +class RefactorValueError(ValueError): + """An unhandled value failure the Python planner also leaves to the caller.""" + + +class RefactorIOError(OSError): + """A file read or write failure the Python planner also leaves to the caller.""" diff --git a/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py b/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py deleted file mode 100644 index 8b8ec24ce7..0000000000 --- a/src/sqlbuild/compiler/refactoring/main/commit_refactor_plan.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Write a verified refactoring plan into the project.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.refactoring._helpers.project.workspace import commit_changes -from sqlbuild.compiler.refactoring.models import RefactorPlan - - -def commit_refactor_plan( - *, project_dir: Path, originals: dict[str, str], plan: RefactorPlan -) -> tuple[Path, ...]: - """Write every change, restoring all files if any write fails, and return written paths.""" - - return commit_changes(project_dir=project_dir, originals=originals, changes=plan.changes) diff --git a/src/sqlbuild/compiler/refactoring/main/find_python_string_locations.py b/src/sqlbuild/compiler/refactoring/main/find_python_string_locations.py new file mode 100644 index 0000000000..76168ab6cc --- /dev/null +++ b/src/sqlbuild/compiler/refactoring/main/find_python_string_locations.py @@ -0,0 +1,34 @@ +"""Find Python strings that name a model or column a refactoring renames.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.refactoring._helpers.python_strings import python_paths, string_locations +from sqlbuild.compiler.refactoring.models import ManualLocation + + +def find_python_string_locations( + *, + project_dir: Path, + discovered: DiscoveredProjectInputs, + names: tuple[str, ...], + reason: str, + context: str | None = None, +) -> tuple[ManualLocation, ...]: + """Flag Python strings that name a model or column; with context, only SQL naming it too.""" + + locations: list[ManualLocation] = [] + relative_path: Path + for relative_path in python_paths(discovered=discovered): + locations.extend( + string_locations( + project_dir=project_dir, + relative_path=relative_path, + names=names, + reason=reason, + context=context, + ) + ) + return tuple(locations) diff --git a/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py b/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py deleted file mode 100644 index 40c92ac8ba..0000000000 --- a/src/sqlbuild/compiler/refactoring/main/plan_column_rename.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Plan a column rename from compiler facts, one step or cascading through pass-throughs.""" - -from __future__ import annotations - -from sqlbuild.compiler.refactoring._helpers.columns.column_planning import ( - column_rename_context, - column_rename_parts, -) -from sqlbuild.compiler.refactoring._helpers.text.text_edits import build_plan -from sqlbuild.compiler.refactoring.constants import COLUMN_MANUAL_HELP -from sqlbuild.compiler.refactoring.models import ( - ColumnRenameContext, - RefactorParts, - RefactorPlan, - RefactorProject, - RefactorRequest, -) - - -def plan_column_rename(*, project: RefactorProject, request: RefactorRequest) -> RefactorPlan: - """Plan every edit that renames one output column and its downstream references.""" - - context: ColumnRenameContext = column_rename_context(project=project, request=request) - parts: RefactorParts = column_rename_parts(context=context) - return build_plan( - request=request, - parts=parts, - moves={}, - renamed_columns=tuple((model, context.old, context.new) for model in parts.cascaded), - help=COLUMN_MANUAL_HELP, - ) diff --git a/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py b/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py deleted file mode 100644 index 9770d28eb5..0000000000 --- a/src/sqlbuild/compiler/refactoring/main/plan_model_refactor.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Plan a model rename or move from compiler facts.""" - -from __future__ import annotations - -from sqlbuild.compiler.refactoring._helpers.renames.model_planning import model_parts, model_target -from sqlbuild.compiler.refactoring._helpers.text.text_edits import build_plan -from sqlbuild.compiler.refactoring.constants import MODEL_MANUAL_HELP -from sqlbuild.compiler.refactoring.models import ( - ModelTarget, - RefactorParts, - RefactorPlan, - RefactorProject, - RefactorRequest, -) - - -def plan_model_refactor(*, project: RefactorProject, request: RefactorRequest) -> RefactorPlan: - """Plan every edit that renames or moves one model, without touching files.""" - - target: ModelTarget = model_target(project=project, request=request) - parts: RefactorParts = model_parts(project=project, target=target) - return build_plan( - request=target.request, - parts=parts, - moves={ - **dict(parts.moves), - **( - {target.source_path: target.destination} - if target.destination != target.source_path - else {} - ), - }, - help=MODEL_MANUAL_HELP, - ) diff --git a/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py b/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py deleted file mode 100644 index c0aa7be537..0000000000 --- a/src/sqlbuild/compiler/refactoring/main/stage_refactor_plan.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Stage a refactoring plan in a scratch copy of the project.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.refactoring._helpers.project.workspace import ( - copy_project_inputs, - read_originals, - write_staged_changes, -) -from sqlbuild.compiler.refactoring.models import RefactorPlan - - -def stage_refactor_plan( - *, project_dir: Path, staging_dir: Path, plan: RefactorPlan, copy_inputs: bool = True -) -> dict[str, str]: - """Copy the project inputs once, apply the plan there, and return the original texts.""" - - if copy_inputs: - _ = copy_project_inputs(project_dir=project_dir, staging_dir=staging_dir) - originals: dict[str, str] = read_originals(project_dir=project_dir, changes=plan.changes) - _ = write_staged_changes(staging_dir=staging_dir, originals=originals, changes=plan.changes) - return originals diff --git a/src/sqlbuild/compiler/refactoring/main/with_model_migration.py b/src/sqlbuild/compiler/refactoring/main/with_model_migration.py deleted file mode 100644 index 2b9feb50f5..0000000000 --- a/src/sqlbuild/compiler/refactoring/main/with_model_migration.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Add model migrate_from to a staged rename whenever its relation moves.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.refactoring._helpers.renames.model_planning import decide_model_migration -from sqlbuild.compiler.refactoring._helpers.text.header_edits import insert_header_entry_edit -from sqlbuild.compiler.refactoring.constants import MIGRATE_FROM_KEY -from sqlbuild.compiler.refactoring.models import ( - FileChange, - ManualLocation, - MigrationDeclaration, - ModelMigrationDecision, - RefactorPlan, - TextEdit, -) - - -def with_model_migration( - *, - plan: RefactorPlan, - before: CompiledProject, - after: CompiledProject | None, - originals: dict[str, str], -) -> RefactorPlan: - """Return the plan with `migrate_from ` on the model whenever its relation moves.""" - - old: str = plan.request.model_name - new: str = plan.request.new_name - decision: ModelMigrationDecision = decide_model_migration( - before=before, after=after, old=old, new=new - ) - change: FileChange | None = next( - (item for item in plan.changes if item.path == plan.request.destination), None - ) - if decision.blocked: - return replace( - plan, - blocking=( - *plan.blocking, - ManualLocation( - path=plan.request.destination or "", - line=None, - column=None, - reason=decision.reason, - ), - ), - ) - if not decision.needed or change is None: - return plan - declaration: str = f"{MIGRATE_FROM_KEY} {old}" - edit: TextEdit | None = insert_header_entry_edit( - contents=originals[change.original_path], entry=declaration, display=declaration - ) - if edit is None: - return replace( - plan, - blocking=( - *plan.blocking, - ManualLocation( - path=change.path, - line=1, - column=1, - reason=f"add {declaration} to the MODEL header to keep history", - ), - ), - ) - return replace( - plan, - changes=tuple( - replace(item, edits=(*item.edits, edit)) if item is change else item - for item in plan.changes - ), - migrations=( - *plan.migrations, - MigrationDeclaration(model_name=new, declaration=declaration, reason=decision.reason), - ), - ) diff --git a/src/sqlbuild/compiler/refactoring/models.py b/src/sqlbuild/compiler/refactoring/models.py index 02c1c82f27..b732582931 100644 --- a/src/sqlbuild/compiler/refactoring/models.py +++ b/src/sqlbuild/compiler/refactoring/models.py @@ -5,19 +5,11 @@ from dataclasses import dataclass, field from pathlib import Path -from yaml.nodes import ScalarNode - -from sqlbuild.compiler.compile.models import CompiledModel, ExpansionSpan from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.refactoring.types import ( EditKind, - HeaderTokenKind, - OffsetLocator, RefactorOperation, - ResourceColumns, - SpanMapper, - SqlFileRole, ) @@ -42,19 +34,6 @@ class RefactorRequest: cascade: bool = False -@dataclass(frozen=True) -class ModelTarget: - """The model a request names, and the request with its name and destination resolved.""" - - model: CompiledModel - request: RefactorRequest - source_path: str - - @property - def destination(self) -> str: - return self.request.destination or self.source_path - - @dataclass(frozen=True) class TextEdit: """One replacement in an authored file, in offsets of the file before any edit.""" @@ -112,206 +91,3 @@ class RefactorPlan: migrations: tuple[MigrationDeclaration, ...] = field(default_factory=tuple) renamed_columns: tuple[tuple[str, str, str], ...] = field(default_factory=tuple) help: str | None = None - - -@dataclass(frozen=True) -class RefactorParts: - """Edits and findings from one planning step, merged into a plan at the end.""" - - edits: tuple[tuple[str, TextEdit], ...] = field(default_factory=tuple) - manual: tuple[ManualLocation, ...] = field(default_factory=tuple) - blocking: tuple[ManualLocation, ...] = field(default_factory=tuple) - migrations: tuple[MigrationDeclaration, ...] = field(default_factory=tuple) - cascaded: tuple[str, ...] = field(default_factory=tuple) - moves: tuple[tuple[str, str], ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ProjectSqlFile: - """One authored SQL file and the role discovery gave it.""" - - relative_path: str - contents: str - role: SqlFileRole - - -@dataclass(frozen=True) -class AuthoredBody: - """One SQL body inside an authored non-model file.""" - - path: str - role: SqlFileRole - contents: str - start: int - text: str - - -@dataclass(frozen=True) -class ResourceSite: - """One `__ref`, `__source`, or `__seed` call written in authored SQL.""" - - kind: str - name: str - start: int - end: int - name_start: int - name_end: int - - -@dataclass(frozen=True) -class AnalysisSql: - """SQL whose interpolation sites were replaced by same-length identifiers.""" - - sql: str - tables: dict[tuple[str, str], frozenset[str]] = field(default_factory=dict) - - -@dataclass(frozen=True) -class ModelBody: - """A model's compiled query and how it maps onto the authored file.""" - - relative_path: str - contents: str - body_start: int - compiled_sql: str - passes: tuple[tuple[ExpansionSpan, ...], ...] - - -@dataclass(frozen=True) -class HeaderToken: - """One MODEL header token in file offsets, with its nesting depth.""" - - kind: HeaderTokenKind - value: str - start: int - end: int - depth: int - - -@dataclass(frozen=True) -class RelationshipTokens: - """The target and field value tokens of one relationships audit in a MODEL header.""" - - target: HeaderToken | None - field: HeaderToken | None - called: bool - - -@dataclass(frozen=True) -class YamlRelationship: - """The `to` and `field` scalars of one relationships audit in a YAML declaration.""" - - target: ScalarNode | None - field: ScalarNode | None - - -@dataclass(frozen=True) -class HeaderEntry: - """One top-level `key value` entry of a MODEL header.""" - - key: str - tokens: tuple[HeaderToken, ...] - - -@dataclass(frozen=True) -class ColumnReference: - """One column expression that resolves to the renamed column.""" - - start: int - end: int - name_start: int - name_end: int - scope: str - projected: bool - alias: str | None - alias_start: int | None - alias_end: int | None - - -@dataclass(frozen=True) -class ColumnSite: - """A scoped construct at an optional span.""" - - scope: str - start: int | None - end: int | None - - -@dataclass(frozen=True) -class OutputColumn: - """One projection that names the column in a requested output scope.""" - - scope: str - start: int | None - end: int | None - alias_start: int | None - alias_end: int | None - - -@dataclass(frozen=True) -class ColumnFacts: - """Every rename-relevant fact about one column in one SQL body.""" - - parsed: bool - references: tuple[ColumnReference, ...] = field(default_factory=tuple) - stars: tuple[ColumnSite, ...] = field(default_factory=tuple) - joins: tuple[ColumnSite, ...] = field(default_factory=tuple) - unresolved: tuple[ColumnSite, ...] = field(default_factory=tuple) - outputs: tuple[OutputColumn, ...] = field(default_factory=tuple) - star_outputs: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class ColumnQuery: - """Which column of which relations one analysis resolves.""" - - column: str - target_tables: frozenset[str] - target_ctes: frozenset[str] = frozenset() - output_scopes: frozenset[str] = frozenset() - - -@dataclass(frozen=True) -class BodyContext: - """Where one analysed SQL body lives, and how its offsets map onto the file.""" - - path: str - contents: str - map_span: SpanMapper - locate: OffsetLocator - fallback_offset: int - - -@dataclass(frozen=True) -class BodyEdits: - """Edits, manual locations, and root pass-throughs found in one SQL body.""" - - edits: tuple[TextEdit, ...] = field(default_factory=tuple) - manual: tuple[ManualLocation, ...] = field(default_factory=tuple) - passes_through: bool = False - - -@dataclass(frozen=True) -class ColumnRenameContext: - """Everything a column rename reads while it walks downstream models.""" - - project: RefactorProject - owner: CompiledModel - dialect: str - columns: ResourceColumns - contents: dict[str, str] - bodies: tuple[AuthoredBody, ...] - yaml_files: tuple[ProjectSqlFile, ...] - schema_files: tuple[ProjectSqlFile, ...] - old: str - new: str - cascade: bool - - -@dataclass(frozen=True) -class ModelMigrationDecision: - """Whether a model needs migrate_from, and why, or why none can be written.""" - - needed: bool - reason: str - blocked: bool = False diff --git a/src/sqlbuild/compiler/refactoring/types.py b/src/sqlbuild/compiler/refactoring/types.py index 9377e1f1ed..f92d584113 100644 --- a/src/sqlbuild/compiler/refactoring/types.py +++ b/src/sqlbuild/compiler/refactoring/types.py @@ -2,9 +2,7 @@ from __future__ import annotations -from collections.abc import Callable from enum import StrEnum -from typing import Protocol class RefactorOperation(StrEnum): @@ -24,6 +22,16 @@ class RefactorStatus(StrEnum): COMPILE_FAILED = "compile_failed" +class NativeRefactorErrorKind(StrEnum): + """Which exception a native refactoring failure becomes.""" + + INPUT = "input" + EDIT = "edit" + WRITE = "write" + VALUE = "value" + IO = "io" + + class EditKind(StrEnum): """What one text edit changes, for output grouping.""" @@ -45,23 +53,3 @@ class SqlFileRole(StrEnum): FUNCTION = "function" SCHEMA = "schema" YAML = "yaml" - - -class HeaderTokenKind(StrEnum): - """MODEL header token kinds.""" - - END = "end" - WORD = "word" - STRING = "string" - SYMBOL = "symbol" - - -type ResourceColumns = dict[tuple[str, str], tuple[str, ...]] -type SpanMapper = Callable[[int, int], tuple[int, int] | None] -type OffsetLocator = Callable[[int], int] - - -class NativeColumnReferences(Protocol): - """Native column-reference analysis boundary.""" - - def analyze_column_references_json(self, request_json: str) -> str: ... diff --git a/src/sqlbuild/compiler/references/_helpers/resolution.py b/src/sqlbuild/compiler/references/_helpers/resolution.py index 5e0a1fae62..58f7338282 100644 --- a/src/sqlbuild/compiler/references/_helpers/resolution.py +++ b/src/sqlbuild/compiler/references/_helpers/resolution.py @@ -2,87 +2,32 @@ from __future__ import annotations -import re - -from sqlbuild.compiler.references._helpers.patterns import reference_call_prefix_pattern_text +import sqlbuild._native as _native +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment -from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment -from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import skip_quoted_text -_UNRESOLVED_REF_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.REF) -) -_UNRESOLVED_SEED_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.SEED) -) -_UNRESOLVED_SOURCE_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.SOURCE) -) -_UNRESOLVED_UDF_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.UDF) +_MARKER_KINDS: tuple[tuple[SqlReferenceKind, str], ...] = ( + (SqlReferenceKind.REF, "R001"), + (SqlReferenceKind.SEED, "R002"), + (SqlReferenceKind.SOURCE, "R003"), + (SqlReferenceKind.UDF, "R004"), + (SqlReferenceKind.TABLE_FUNCTION, "R005"), ) -_UNRESOLVED_TABLE_FUNCTION_PATTERN: re.Pattern[str] = re.compile( - reference_call_prefix_pattern_text(SqlReferenceKind.TABLE_FUNCTION) -) -_SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`", "$"}) def assert_no_unresolved_sql_markers(*, sql: str, context: str) -> None: """Fail fast if executable SQL still contains unresolved ref/source markers.""" - if _contains_executable_marker(sql=sql, pattern=_UNRESOLVED_REF_PATTERN, context=context): - raise _coded_value_error( - message=f"{context} still contains unresolved " - f"{SqlReferenceKind.REF.placeholder_call()} markers", - code="R001", - ) - if _contains_executable_marker(sql=sql, pattern=_UNRESOLVED_SEED_PATTERN, context=context): - raise _coded_value_error( - message=f"{context} still contains unresolved " - f"{SqlReferenceKind.SEED.placeholder_call()} markers", - code="R002", - ) - if _contains_executable_marker(sql=sql, pattern=_UNRESOLVED_SOURCE_PATTERN, context=context): - raise _coded_value_error( - message=f"{context} still contains unresolved " - f"{SqlReferenceKind.SOURCE.placeholder_call()} markers", - code="R003", - ) - if _contains_executable_marker(sql=sql, pattern=_UNRESOLVED_UDF_PATTERN, context=context): - raise _coded_value_error( - message=f"{context} still contains unresolved " - f"{SqlReferenceKind.UDF.placeholder_call()} markers", - code="R004", - ) - if _contains_executable_marker( - sql=sql, pattern=_UNRESOLVED_TABLE_FUNCTION_PATTERN, context=context - ): - raise _coded_value_error( - message=f"{context} still contains unresolved " - f"{SqlReferenceKind.TABLE_FUNCTION.placeholder_call()} markers", - code="R005", - ) - - -def _contains_executable_marker(*, sql: str, pattern: re.Pattern[str], context: str) -> bool: - if pattern.search(sql) is None: - return False - index: int = 0 - while index < len(sql): - if sql[index] in _SQL_QUOTE_TOKENS: - index = skip_quoted_text(sql=sql, start=index, context=context) - continue - if sql.startswith("--", index): - index = skip_line_comment(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment(sql=sql, start=index, context=context) - continue - if pattern.match(sql, index) is not None: - return True - index += 1 - return False + marker, unclosed = _native.unresolved_reference_marker_kind(sql, context) + if unclosed is not None: + raise CompileInputError(unclosed) + if marker is None: + return + kind, code = _MARKER_KINDS[marker] + raise _coded_value_error( + message=f"{context} still contains unresolved {kind.placeholder_call()} markers", + code=code, + ) def _coded_value_error(*, message: str, code: str) -> ValueError: diff --git a/src/sqlbuild/compiler/references/main/reference_call_prefix_pattern_text.py b/src/sqlbuild/compiler/references/main/_reference_call_prefix_pattern_text.py similarity index 100% rename from src/sqlbuild/compiler/references/main/reference_call_prefix_pattern_text.py rename to src/sqlbuild/compiler/references/main/_reference_call_prefix_pattern_text.py diff --git a/src/sqlbuild/compiler/resource_names/_helpers/__init__.py b/src/sqlbuild/compiler/resource_names/_helpers/__init__.py deleted file mode 100644 index 30925007f7..0000000000 --- a/src/sqlbuild/compiler/resource_names/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Private resource name helpers.""" diff --git a/src/sqlbuild/compiler/resource_names/_helpers/spelling.py b/src/sqlbuild/compiler/resource_names/_helpers/spelling.py deleted file mode 100644 index d0623119e4..0000000000 --- a/src/sqlbuild/compiler/resource_names/_helpers/spelling.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Canonical resource identity spelling helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.resource_names.constants import ( - ACRONYM_BOUNDARY_PATTERN, - INVALID_CHARACTER_PATTERN, - UPPERCASE_BOUNDARY_PATTERN, -) - - -def suggest_resource_identity(*, name: str, private_identity: bool) -> str: - """Return the suggested canonical spelling for an invalid identity.""" - - with_acronym_boundaries: str = ACRONYM_BOUNDARY_PATTERN.sub("_", name.strip()) - with_word_boundaries: str = UPPERCASE_BOUNDARY_PATTERN.sub("_", with_acronym_boundaries) - replaced: str = INVALID_CHARACTER_PATTERN.sub("_", with_word_boundaries).lower() - corrected: str = replaced.strip("_") or "resource_name" - return f"_{corrected}" if private_identity else corrected diff --git a/src/sqlbuild/compiler/resource_names/constants.py b/src/sqlbuild/compiler/resource_names/constants.py deleted file mode 100644 index ff8285ea60..0000000000 --- a/src/sqlbuild/compiler/resource_names/constants.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Resource name validation constants.""" - -from __future__ import annotations - -import re - -SNAKE_CASE_PATTERN: re.Pattern[str] = re.compile(r"^[a-z](?:[a-z0-9_]*[a-z0-9])?$") -PRIVATE_SNAKE_CASE_PATTERN: re.Pattern[str] = re.compile(r"^_[a-z](?:[a-z0-9_]*[a-z0-9])?$") -UPPERCASE_BOUNDARY_PATTERN: re.Pattern[str] = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") -ACRONYM_BOUNDARY_PATTERN: re.Pattern[str] = re.compile(r"(?<=[A-Z])(?=[A-Z][a-z])") -INVALID_CHARACTER_PATTERN: re.Pattern[str] = re.compile(r"[^A-Za-z0-9_]+") diff --git a/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py b/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py index 76f5a4c61a..9181d26aa2 100644 --- a/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py +++ b/src/sqlbuild/compiler/resource_names/main/_validate_resource_identity.py @@ -2,14 +2,9 @@ from __future__ import annotations -import re from pathlib import Path -from sqlbuild.compiler.resource_names._helpers.spelling import suggest_resource_identity -from sqlbuild.compiler.resource_names.constants import ( - PRIVATE_SNAKE_CASE_PATTERN, - SNAKE_CASE_PATTERN, -) +import sqlbuild._native as _native from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError @@ -18,17 +13,11 @@ def validate_resource_identity( ) -> None: """Reject one identity unless it uses canonical lowercase snake_case.""" - pattern: re.Pattern[str] = ( - PRIVATE_SNAKE_CASE_PATTERN if private_identity else SNAKE_CASE_PATTERN - ) - if pattern.fullmatch(name) is not None: + failure: tuple[str, str] | None = _native.resource_identity_suggestion(name, private_identity) + if failure is None: return - corrected_name: str = suggest_resource_identity(name=name, private_identity=private_identity) + corrected_name, help_text = failure raise ResourceIdentityError( f"Invalid {kind} identity '{name}' in {path}; use snake_case '{corrected_name}'", - help=( - "Rename the authored identity and update its references, selectors, and integration " - "keys. SQLBuild does not silently normalize resource identities. Double underscores " - "remain valid." - ), + help=help_text, ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/builder.py b/src/sqlbuild/compiler/scopes/_helpers/builder.py index 454752abb4..898763cd9f 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/builder.py +++ b/src/sqlbuild/compiler/scopes/_helpers/builder.py @@ -1,68 +1,27 @@ -"""Projection of discovered compiler inputs into canonical scope facts.""" +"""Tolerant scope snapshots built by the native scope index from partial discovery.""" from __future__ import annotations -import hashlib -import inspect -from collections import defaultdict -from collections.abc import Mapping -from dataclasses import dataclass, replace +from dataclasses import replace from pathlib import Path -from types import CodeType -from sqlbuild.compiler.compile.main._scope_relationship_grants import ( - build_scope_relationship_grants, -) +from sqlbuild.compiler.compile.constants import SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM from sqlbuild.compiler.compile.main._static_macro_inventory import inventory_project_macros -from sqlbuild.compiler.compile.models import ( - LoadedMacro, - ScopeRelationshipBuild, - StaticMacroExport, - StaticMacroInventory, -) +from sqlbuild.compiler.compile.models import StaticMacroInventory from sqlbuild.compiler.discovery.main._scope_snapshot import discover_scope_snapshot -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredAuditFile, - DiscoveredConstantFile, - DiscoveredEnumFile, - DiscoveredHookFunction, - DiscoveredMacroFile, - DiscoveredModelSchemaFile, - DiscoveredProjectInputs, - DiscoveredSqlHookFile, - DiscoveryFileFault, - EnumDeclaration, - TolerantScopeDiscovery, -) -from sqlbuild.compiler.scopes._helpers.identities import format_identity -from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes.constants import GLOBAL_DECLARATION_DIRECTORIES +from sqlbuild.compiler.discovery.models import DiscoveryFileFault, TolerantScopeDiscovery +from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex +from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.models import ( - ConstantMetadata, - DeclarationIdentity, - DeclarationRecord, - EnumMemberMetadata, - EnumMetadata, - MacroMetadata, OwnershipRoot, ResourceIdentity, ResourceRecord, ScopeCompleteness, ScopeDiagnostic, ScopeIndex, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - OwnershipRootKind, - ResourceKind, - ScopeDiagnosticCode, - ScopeKind, - UsageKind, ) +from sqlbuild.compiler.scopes.types import ResourceKind, ScopeDiagnosticCode from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.sql_values.types import SqlValueKind _ROOTS: dict[ResourceKind, str] = { ResourceKind.MODEL: "models", @@ -71,191 +30,6 @@ ResourceKind.FUNCTION: "functions/sql", ResourceKind.SOURCE: "sources", } -_COLLECTION_KINDS: frozenset[SqlValueKind] = frozenset( - {SqlValueKind.LIST, SqlValueKind.SET, SqlValueKind.OBJECT} -) -_SEED_ROOT: OwnershipRoot = OwnershipRoot("seeds", OwnershipRootKind.GLOBAL, ResourceKind.SEED) -_PYTHON_FUNCTION_ROOT: OwnershipRoot = OwnershipRoot( - "functions/python", OwnershipRootKind.GLOBAL, ResourceKind.FUNCTION -) - - -@dataclass(frozen=True) -class _NamedPlacement: - scope: ScopeKind - ownership_root: OwnershipRoot - owning_path: str | None - - @classmethod - def of( - cls, - item: DiscoveredAuditFile - | DiscoveredModelSchemaFile - | DiscoveredSqlHookFile - | DiscoveredHookFunction, - ) -> _NamedPlacement: - fallback: str = normalize_path(path=item.declaration_root or item.relative_path.parent) - return cls( - scope=item.scope_kind, - ownership_root=_root(path=item.ownership_root, fallback=fallback), - owning_path=( - normalize_path(path=item.owning_path) if item.owning_path is not None else None - ), - ) - - def record(self, *, kind: DeclarationKind, name: str, path: Path) -> DeclarationRecord: - return DeclarationRecord( - identity=DeclarationIdentity(kind, name), - path=normalize_path(path=path), - line=1, - column=1, - scope=self.scope, - ownership_root=self.ownership_root, - owning_path=self.owning_path, - ) - - -def build_index( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: Mapping[str, LoadedMacro] | None, - static_macros: tuple[StaticMacroExport, ...] = (), -) -> ScopeIndex: - resources: list[ResourceRecord] = [] - declarations: list[DeclarationRecord] = [] - - for model_file in discovered_inputs.model_files: - model_name: str = _name_or_stem( - value=model_file.header_values.get("name"), path=model_file.relative_path - ) - resources.append( - _resource_record( - kind=ResourceKind.MODEL, name=model_name, path=model_file.relative_path - ) - ) - owner: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, model_name) - declarations.extend( - _private_declarations( - enums=model_file.enum_declarations, - constants=model_file.constant_declarations, - owner=owner, - path=model_file.relative_path, - ) - ) - for test_file in discovered_inputs.test_files: - for block in test_file.blocks: - resources.append( - _resource_record( - kind=ResourceKind.TEST, - name=block.name or test_file.relative_path.stem, - path=test_file.relative_path, - ) - ) - for scenario_file in discovered_inputs.scenario_files: - resources.append( - _resource_record( - kind=ResourceKind.SCENARIO, - name=scenario_file.name, - path=scenario_file.relative_path, - ) - ) - for function_file in discovered_inputs.sql_function_files: - resources.append( - _resource_record( - kind=ResourceKind.FUNCTION, - name=_name_or_stem( - value=function_file.header_values.get("name"), - path=function_file.relative_path, - ), - path=function_file.relative_path, - ) - ) - for python_function_file in discovered_inputs.python_function_files: - resources.append( - ResourceRecord( - identity=ResourceIdentity( - ResourceKind.FUNCTION, python_function_file.file_path.stem - ), - path=normalize_path(path=python_function_file.relative_path), - ownership_root=_PYTHON_FUNCTION_ROOT, - ) - ) - for seed_file in discovered_inputs.seed_files: - resources.append( - ResourceRecord( - identity=ResourceIdentity(ResourceKind.SEED, seed_file.file_path.stem), - path=normalize_path(path=seed_file.relative_path), - ownership_root=_SEED_ROOT, - ) - ) - for source_file in discovered_inputs.source_files: - for source in source_file.source_entries: - resources.append( - _resource_record( - kind=ResourceKind.SOURCE, - name=source.name, - path=source_file.relative_path, - ) - ) - - for enum_file in discovered_inputs.enum_files: - declarations.extend(_enum_file_records(enum_file)) - for constant_file in discovered_inputs.constant_files: - declarations.extend(_constant_file_records(constant_file)) - declarations.extend(_named_declaration_records(discovered_inputs=discovered_inputs)) - if loaded_macros is not None: - macro_files: dict[str, DiscoveredMacroFile] = { - normalize_path(path=item.relative_path): item for item in discovered_inputs.macro_files - } - for loaded in loaded_macros.values(): - declarations.append(_macro_record(loaded=loaded, files=macro_files)) - else: - macro_files = { - normalize_path(path=item.relative_path): item for item in discovered_inputs.macro_files - } - declarations.extend( - _static_macro_record(item=macro, files=macro_files) for macro in static_macros - ) - - diagnostics: tuple[ScopeDiagnostic, ...] = tuple( - sorted( - (*_resource_diagnostics(resources), *_declaration_diagnostics(declarations)), - key=_diagnostic_key, - ) - ) - usages: tuple[UsageRecord, ...] = _macro_dependency_usages(declarations=declarations) - return ScopeIndex( - ownership_roots=tuple( - OwnershipRoot(path=path, resource_kind=kind) - for kind, path in sorted(_ROOTS.items(), key=lambda item: item[1]) - ), - resources=tuple(sorted(resources, key=_resource_key)), - declarations=tuple(sorted(declarations, key=_declaration_key)), - usages=usages, - diagnostics=diagnostics, - completeness=ScopeCompleteness( - runtime_usage=False, - relationships=False, - placement=False, - promotion_impact=False, - ), - ) - - -def _macro_dependency_usages(*, declarations: list[DeclarationRecord]) -> tuple[UsageRecord, ...]: - usages: list[UsageRecord] = [] - for record in declarations: - if record.macro is None: - continue - for dependency in record.macro.dependencies: - usages.append( - UsageRecord( - consumer=record.identity, - declaration=dependency, - kind=UsageKind.DECLARATION_DEPENDENCY, - ) - ) - return tuple(dict.fromkeys(usages)) def build_tolerant_scope_index( @@ -267,20 +41,21 @@ def build_tolerant_scope_index( macro_inventory: StaticMacroInventory = inventory_project_macros( macro_files=snapshot.discovered_inputs.macro_files ) - index: ScopeIndex = build_index( + native: NativeScopeIndex = open_native_scope_index( discovered_inputs=snapshot.discovered_inputs, - loaded_macros=None, + loaded_macros={}, static_macros=macro_inventory.exports, ) - relationships: ScopeRelationshipBuild = build_scope_relationship_grants( + relationship_faults: tuple[tuple[Path, str], ...] = native.grant_relationships( discovered_inputs=snapshot.discovered_inputs, - index=index, - sql_lexical_syntax=sql_lexical_syntax, + syntax=sql_lexical_syntax, + scan_store=(None, SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM), ) + index: ScopeIndex = native.index_with_relationships() diagnostics: tuple[ScopeDiagnostic, ...] = _tolerant_diagnostics( snapshot=snapshot, macro_inventory=macro_inventory, - relationships=relationships, + relationship_faults=relationship_faults, ) prospective: tuple[ResourceRecord, ...] = tuple( record @@ -290,7 +65,6 @@ def build_tolerant_scope_index( ) return replace( index, - grants=relationships.grants, resources=tuple( sorted( dict.fromkeys((*index.resources, *prospective)), @@ -309,7 +83,7 @@ def build_tolerant_scope_index( snapshot.declaration_faults or snapshot.config_faults or macro_inventory.faults ), runtime_usage=False, - relationships=not snapshot.relationship_faults and not relationships.faults, + relationships=not snapshot.relationship_faults and not relationship_faults, placement=False, promotion_impact=False, ), @@ -320,7 +94,7 @@ def _tolerant_diagnostics( *, snapshot: TolerantScopeDiscovery, macro_inventory: StaticMacroInventory, - relationships: ScopeRelationshipBuild, + relationship_faults: tuple[tuple[Path, str], ...], ) -> tuple[ScopeDiagnostic, ...]: diagnostics: list[ScopeDiagnostic] = [] diagnostics.extend( @@ -350,10 +124,10 @@ def _tolerant_diagnostics( diagnostics.extend( ScopeDiagnostic( ScopeDiagnosticCode.RELATIONSHIP_PARSE_ERROR, - fault.message, - path=fault.relative_path.as_posix(), + message, + path=path.as_posix(), ) - for fault in relationships.faults + for path, message in relationship_faults ) return tuple(diagnostics) @@ -380,380 +154,9 @@ def _prospective_resource(*, path: Path) -> ResourceRecord | None: return None -def _resource_record(*, kind: ResourceKind, name: str, path: Path) -> ResourceRecord: - root: str = _ROOTS[kind] - return ResourceRecord( - identity=ResourceIdentity(kind=kind, name=name), - path=normalize_path(path=path), - ownership_root=OwnershipRoot(path=root, resource_kind=kind), - ) - - -def _name_or_stem(*, value: object, path: Path) -> str: - return value if isinstance(value, str) and value else path.stem - - -def _root(*, path: Path | None, fallback: str, kind: ResourceKind | None = None) -> OwnershipRoot: - if path is None: - return OwnershipRoot(fallback, OwnershipRootKind.GLOBAL) - normalized: str = normalize_path(path=path) - is_global: bool = normalized in GLOBAL_DECLARATION_DIRECTORIES - resource_kind: ResourceKind | None = next( - (item_kind for item_kind, item_path in _ROOTS.items() if item_path == normalized), - None, - ) - return OwnershipRoot( - normalized, - OwnershipRootKind.GLOBAL if is_global else OwnershipRootKind.RESOURCE, - resource_kind=kind or resource_kind, - ) - - -def _enum_file_records(file: DiscoveredEnumFile) -> list[DeclarationRecord]: - return [ - _enum_record( - declaration=item, - scope=file.scope_kind, - ownership_root=_root(path=file.ownership_root, fallback="enums"), - owning_path=file.owning_path, - path=file.relative_path, - ) - for item in file.declarations - ] - - -def _constant_file_records(file: DiscoveredConstantFile) -> list[DeclarationRecord]: - return [ - _constant_record( - declaration=item, - scope=file.scope_kind, - ownership_root=_root(path=file.ownership_root, fallback="constants"), - owning_path=file.owning_path, - path=file.relative_path, - ) - for item in file.declarations - ] - - -def _named_declaration_records( - *, discovered_inputs: DiscoveredProjectInputs -) -> list[DeclarationRecord]: - records: list[DeclarationRecord] = [] - for audit_file in discovered_inputs.audit_files: - placement: _NamedPlacement = _NamedPlacement.of(audit_file) - if audit_file.declaration_kind is DeclarationKind.AUDIT: - records.append( - placement.record( - kind=DeclarationKind.AUDIT, - name=audit_file.relative_path.stem, - path=audit_file.relative_path, - ) - ) - continue - records.extend( - placement.record( - kind=DeclarationKind.SINGULAR_AUDIT, - name=block.name or audit_file.relative_path.stem, - path=audit_file.relative_path, - ) - for block in audit_file.blocks - ) - for schema_file in discovered_inputs.model_schema_files: - placement = _NamedPlacement.of(schema_file) - records.extend( - placement.record( - kind=DeclarationKind.SCHEMA, name=declaration.name, path=schema_file.relative_path - ) - for declaration in schema_file.declarations - ) - for hook_file in discovered_inputs.sql_hook_files: - records.append( - _NamedPlacement.of(hook_file).record( - kind=DeclarationKind.SQL_HOOK, name=hook_file.name, path=hook_file.relative_path - ) - ) - for hook_function in discovered_inputs.hook_functions: - records.append( - _NamedPlacement.of(hook_function).record( - kind=DeclarationKind.PYTHON_HOOK, - name=hook_function.name, - path=hook_function.relative_path, - ) - ) - return records - - -def _private_declarations( - *, - enums: tuple[EnumDeclaration, ...], - constants: tuple[ConstantDeclaration, ...], - owner: ResourceIdentity, - path: Path, -) -> list[DeclarationRecord]: - root: OwnershipRoot = OwnershipRoot("models", resource_kind=ResourceKind.MODEL) - return [ - _enum_record( - declaration=item, - scope=ScopeKind.PRIVATE, - ownership_root=root, - owning_path=path.parent, - path=path, - owner=owner, - ) - for item in enums - ] + [ - _constant_record( - declaration=item, - scope=ScopeKind.PRIVATE, - ownership_root=root, - owning_path=path.parent, - path=path, - owner=owner, - ) - for item in constants - ] - - -def _enum_record( - *, - declaration: EnumDeclaration, - scope: ScopeKind, - ownership_root: OwnershipRoot, - owning_path: Path | None, - path: Path, - owner: ResourceIdentity | None = None, -) -> DeclarationRecord: - return DeclarationRecord( - identity=DeclarationIdentity(DeclarationKind.ENUM, declaration.name, owner), - path=normalize_path(path=path), - line=1, - column=1, - scope=scope, - ownership_root=ownership_root, - owning_path=normalize_path(path=owning_path) if owning_path is not None else None, - enum=enum_metadata(declaration), - ) - - -def _constant_record( - *, - declaration: ConstantDeclaration, - scope: ScopeKind, - ownership_root: OwnershipRoot, - owning_path: Path | None, - path: Path, - owner: ResourceIdentity | None = None, -) -> DeclarationRecord: - return DeclarationRecord( - identity=DeclarationIdentity(DeclarationKind.CONSTANT, declaration.name, owner), - path=normalize_path(path=path), - line=1, - column=1, - scope=scope, - ownership_root=ownership_root, - owning_path=normalize_path(path=owning_path) if owning_path is not None else None, - constant=constant_metadata(declaration), - ) - - -def _macro_record( - *, loaded: LoadedMacro, files: dict[str, DiscoveredMacroFile] -) -> DeclarationRecord: - path: str = normalize_path(path=loaded.relative_path) - discovered: DiscoveredMacroFile | None = files.get(path) - scope: ScopeKind = discovered.scope_kind if discovered is not None else ScopeKind.GLOBAL - ownership_root: OwnershipRoot = _root( - path=discovered.ownership_root if discovered is not None else None, - fallback="macros", - ) - owning_path: str | None = ( - normalize_path(path=discovered.owning_path) - if discovered is not None and discovered.owning_path is not None - else None - ) - return DeclarationRecord( - identity=DeclarationIdentity(DeclarationKind.MACRO, loaded.name), - path=path, - line=macro_line(loaded), - column=1, - scope=scope, - ownership_root=ownership_root, - owning_path=owning_path, - macro=macro_metadata(loaded), - ) - - -def _static_macro_record( - *, item: StaticMacroExport, files: dict[str, DiscoveredMacroFile] -) -> DeclarationRecord: - path: str = normalize_path(path=item.relative_path) - discovered: DiscoveredMacroFile | None = files.get(path) - return DeclarationRecord( - identity=DeclarationIdentity(DeclarationKind.MACRO, item.name), - path=path, - line=item.line, - column=1, - scope=discovered.scope_kind if discovered is not None else ScopeKind.GLOBAL, - ownership_root=_root( - path=discovered.ownership_root if discovered is not None else None, - fallback="macros", - ), - owning_path=( - normalize_path(path=discovered.owning_path) - if discovered is not None and discovered.owning_path is not None - else None - ), - macro=MacroMetadata( - parameters=item.parameters, - dependencies=item.dependencies, - source_digest=item.source_digest, - ), - ) - - -def _declaration_diagnostics(records: list[DeclarationRecord]) -> tuple[ScopeDiagnostic, ...]: - diagnostics: list[ScopeDiagnostic] = [] - public: dict[tuple[DeclarationKind, str], list[DeclarationRecord]] = defaultdict(list) - private: dict[DeclarationIdentity, list[DeclarationRecord]] = defaultdict(list) - for record in records: - name: str = record.identity.name - if name.startswith("__"): - diagnostics.append( - _diagnostic( - record=record, - code=ScopeDiagnosticCode.RESERVED_DECLARATION_NAME, - message=f"Declaration name '{name}' uses reserved '__' prefix", - ) - ) - elif record.scope is ScopeKind.PRIVATE and ( - not name.startswith("_") or name[1:].startswith("_") - ): - diagnostics.append( - _diagnostic( - record=record, - code=ScopeDiagnosticCode.INVALID_DECLARATION_NAME, - message=( - f"Private declaration name '{name}' must have exactly one leading " - "underscore" - ), - ) - ) - elif record.scope is not ScopeKind.PRIVATE and name.startswith("_"): - diagnostics.append( - _diagnostic( - record=record, - code=ScopeDiagnosticCode.INVALID_DECLARATION_NAME, - message=f"Public declaration name '{name}' must not start with underscore", - ) - ) - if record.identity.owner is None: - public[(record.identity.kind, name)].append(record) - else: - private[record.identity].append(record) - for duplicates in (*public.values(), *private.values()): - if len(duplicates) <= 1: - continue - locations: str = ", ".join( - f"{item.path}:{item.line}:{item.column}" - for item in sorted(duplicates, key=_declaration_key) - ) - for record in duplicates: - identity: str = format_identity(identity=record.identity) - diagnostics.append( - _diagnostic( - record=record, - code=ScopeDiagnosticCode.DUPLICATE_DECLARATION, - message=f"Duplicate declaration '{identity}' at {locations}", - ) - ) - return tuple(sorted(diagnostics, key=_diagnostic_key)) - - -def _resource_diagnostics(records: list[ResourceRecord]) -> tuple[ScopeDiagnostic, ...]: - grouped: dict[ResourceIdentity, list[ResourceRecord]] = defaultdict(list) - diagnostics: list[ScopeDiagnostic] = [] - for record in records: - grouped[record.identity].append(record) - for identity, duplicates in grouped.items(): - if len(duplicates) <= 1: - continue - locations: str = ", ".join(item.path for item in sorted(duplicates, key=_resource_key)) - identity_text: str = format_identity(identity=identity) - for record in duplicates: - diagnostics.append( - ScopeDiagnostic( - ScopeDiagnosticCode.DUPLICATE_RESOURCE, - f"Duplicate resource '{identity_text}' at {locations}", - path=record.path, - line=1, - column=1, - resource=record.identity, - ) - ) - return tuple(sorted(diagnostics, key=_diagnostic_key)) - - -def _diagnostic( - *, record: DeclarationRecord, code: ScopeDiagnosticCode, message: str -) -> ScopeDiagnostic: - return ScopeDiagnostic( - code, - message, - path=record.path, - line=record.line, - column=record.column, - declaration=record.identity, - ) - - -def _resource_key(record: ResourceRecord) -> tuple[str, str]: - return (record.path, format_identity(identity=record.identity)) - - -def _declaration_key(record: DeclarationRecord) -> tuple[str, str, int, int]: - return (format_identity(identity=record.identity), record.path, record.line, record.column) +def _resource_key(record: ResourceRecord) -> tuple[str, str, str]: + return (record.path, record.identity.kind.value, record.identity.name) def _diagnostic_key(item: ScopeDiagnostic) -> tuple[str, int, int, str, str]: return (item.path or "", item.line or 0, item.column or 0, item.code.value, item.message) - - -def enum_metadata(declaration: EnumDeclaration) -> EnumMetadata: - """Return the member names and scalar type of one enum.""" - - return EnumMetadata( - members=tuple(EnumMemberMetadata(item.name) for item in declaration.members), - scalar_type=declaration.scalar_type, - ) - - -def constant_metadata(declaration: ConstantDeclaration) -> ConstantMetadata: - """Return the logical type and collection shape of one constant.""" - - is_collection: bool = declaration.value.kind in _COLLECTION_KINDS - payload: object = declaration.value.value - item_count: int | None = len(payload) if is_collection and isinstance(payload, tuple) else None - return ConstantMetadata( - logical_type=declaration.logical_type.display_name, - collection_kind=declaration.value.kind.value if is_collection else None, - item_count=item_count, - nullable=declaration.value.kind is SqlValueKind.NULL, - render_as=declaration.render_as.value if declaration.render_as is not None else None, - ) - - -def macro_metadata(loaded: LoadedMacro) -> MacroMetadata: - """Return the parameters, dependencies and source digest of one loaded macro.""" - - return MacroMetadata( - parameters=tuple(inspect.signature(loaded.function).parameters), - dependencies=loaded.dependencies, - source_digest=hashlib.sha256(loaded.raw_source.encode()).hexdigest(), - ) - - -def macro_line(loaded: LoadedMacro) -> int: - """Return the first line of a loaded macro's function, or 1 without code.""" - - code: object = getattr(loaded.function, "__code__", None) - return code.co_firstlineno if isinstance(code, CodeType) else 1 diff --git a/src/sqlbuild/compiler/scopes/_helpers/cache.py b/src/sqlbuild/compiler/scopes/_helpers/cache.py index 11cf2a132b..ef413b0e97 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/cache.py +++ b/src/sqlbuild/compiler/scopes/_helpers/cache.py @@ -1,42 +1,25 @@ -"""Deterministic fingerprinting and persistent canonical scope-index caching.""" +"""Persistent canonical scope-index caching over the native fingerprint and codec.""" from __future__ import annotations -import hashlib -import json -import os -import tempfile -import tomllib from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from typing import cast +from typing import Any +import sqlbuild._native as _native from sqlbuild.compiler.discovery.constants import ( CANONICAL_AUTHORED_ROOTS, LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, ) from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.compiler.scopes._helpers.lookup import build_lookup -from sqlbuild.compiler.scopes._helpers.paths import normalize_path +from sqlbuild.compiler.scopes._helpers.placement import qualified_row, root_row from sqlbuild.compiler.scopes.constants import ( GLOBAL_DECLARATION_DIRECTORIES, GLOBAL_NAMED_DECLARATION_DIRECTORIES, SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME, - SCOPE_CACHE_MAX_BYTES, - SCOPE_CACHE_SCHEMA_VERSION, - SCOPE_ENUM_MEMBER_FIELDS, - SCOPE_FINGERPRINT_ALGORITHM_VERSION, - SCOPE_LOCAL_CONFIG_KEYS, - SCOPE_MACRO_SUFFIX, - SCOPE_PROJECT_CONFIG_KEYS, - SCOPE_RELATIONSHIP_ROOTS, - SCOPE_SOURCE_SUFFIXES, - SCOPE_TARGET_VARS_KEY, - SCOPE_TARGETS_KEY, ) -from sqlbuild.compiler.scopes.exceptions import ScopeCacheDecodeError from sqlbuild.compiler.scopes.models import ( ConstantMetadata, DeclarationIdentity, @@ -60,10 +43,9 @@ DiagnosticSeverity, GrantKind, InaccessibleReason, - JsonObject, + NativeQualifiedRow, OwnershipRootKind, ResourceKind, - ScopeCacheIdentityType, ScopeDiagnosticCode, ScopeKind, UsageKind, @@ -76,615 +58,276 @@ ".sqlbuildignore", ".gitignore", ) +_SOURCE_ROOTS: tuple[str, ...] = ( + *(Path(*parts).as_posix() for parts in CANONICAL_AUTHORED_ROOTS), + *GLOBAL_DECLARATION_DIRECTORIES, + *GLOBAL_NAMED_DECLARATION_DIRECTORIES, + "python", + "seeds", +) +_RESOURCE_KINDS: frozenset[str] = frozenset(kind.value for kind in ResourceKind) -def encode_scope_index(*, index: ScopeIndex) -> bytes: - """Encode a canonical ScopeIndex payload as deterministic ASCII JSON.""" - - canonical: ScopeIndex = build_lookup(index=index).index - return json.dumps( - _index_payload(canonical), - ensure_ascii=True, - separators=(",", ":"), - sort_keys=True, - ).encode("ascii") - - -def decode_scope_index(*, payload: object) -> ScopeIndex: - """Decode a validated cache payload into immutable scope records.""" - - value: JsonObject = _object(payload) - return ScopeIndex( - ownership_roots=tuple(_root(item) for item in _list(value=value, key="ownership_roots")), - resources=tuple(_resource(item) for item in _list(value=value, key="resources")), - declarations=tuple(_declaration(item) for item in _list(value=value, key="declarations")), - usages=tuple(_usage(item) for item in _list(value=value, key="usages")), - grants=tuple(_grant(item) for item in _list(value=value, key="grants")), - visibility=tuple(_visibility(item) for item in _list(value=value, key="visibility")), - inaccessible=tuple(_inaccessible(item) for item in _list(value=value, key="inaccessible")), - diagnostics=tuple(_diagnostic(item) for item in _list(value=value, key="diagnostics")), - completeness=_completeness(value.get("completeness")), - ) - +def scope_index_fingerprint(*, project_dir: Path) -> str | None: + """Hash canonical scope-relevant authored inputs, or None when they cannot be read.""" -def _index_payload(index: ScopeIndex) -> JsonObject: - return { - "ownership_roots": [_root_payload(item) for item in index.ownership_roots], - "resources": [_resource_payload(item) for item in index.resources], - "declarations": [_declaration_payload(item) for item in index.declarations], - "usages": [_usage_payload(item) for item in index.usages], - "grants": [_grant_payload(item) for item in index.grants], - "visibility": [_visibility_payload(item) for item in index.visibility], - "inaccessible": [_inaccessible_payload(item) for item in index.inaccessible], - "diagnostics": [_diagnostic_payload(item) for item in index.diagnostics], - "completeness": { - "discovery": index.completeness.discovery, - "static_visibility": index.completeness.static_visibility, - "runtime_usage": index.completeness.runtime_usage, - "relationships": index.completeness.relationships, - "placement": index.completeness.placement, - "promotion_impact": index.completeness.promotion_impact, - }, - } - - -def _identity_payload(item: ResourceIdentity | DeclarationIdentity | None) -> object: - if item is None: - return None - if isinstance(item, ResourceIdentity): - return { - "type": ScopeCacheIdentityType.RESOURCE.value, - "kind": item.kind.value, - "name": item.name, - } - return { - "type": ScopeCacheIdentityType.DECLARATION.value, - "kind": item.kind.value, - "name": item.name, - "owner": _identity_payload(item.owner), - } - - -def _identity(payload: object) -> ResourceIdentity | DeclarationIdentity: - item: JsonObject = _object(payload) - if item.get("type") == ScopeCacheIdentityType.RESOURCE: - return ResourceIdentity( - ResourceKind(_string(value=item, key="kind")), - _string(value=item, key="name"), - ) - if item.get("type") != ScopeCacheIdentityType.DECLARATION: - raise ScopeCacheDecodeError("Invalid cache identity type") - owner_payload: object = item.get("owner") - owner: ResourceIdentity | None = None - if owner_payload is not None: - decoded: ResourceIdentity | DeclarationIdentity = _identity(owner_payload) - if not isinstance(decoded, ResourceIdentity): - raise ScopeCacheDecodeError("Invalid cached declaration owner") - owner = decoded - return DeclarationIdentity( - DeclarationKind(_string(value=item, key="kind")), - _string(value=item, key="name"), - owner, + return _native.scope_index_fingerprint( + project_dir, + ( + list(_CONFIG_FILES), + list(_SOURCE_ROOTS), + sorted(GLOBAL_DECLARATION_DIRECTORIES), + sorted(GLOBAL_NAMED_DECLARATION_DIRECTORIES), + _sqlbuild_version(), + ), ) -def _root_payload(item: OwnershipRoot) -> JsonObject: - return { - "path": normalize_path(path=item.path), - "kind": item.kind.value, - "resource_kind": item.resource_kind.value if item.resource_kind is not None else None, - } - +def read_cached_scope_index(*, project_dir: Path, fingerprint: str) -> ScopeIndex | None: + """Read a verified index; every storage or validation fault is a cache miss.""" -def _root(payload: object) -> OwnershipRoot: - item: JsonObject = _object(payload) - resource_kind: object = item.get("resource_kind") - return OwnershipRoot( - path=normalize_path(path=_string(value=item, key="path")), - kind=OwnershipRootKind(_string(value=item, key="kind")), - resource_kind=ResourceKind(resource_kind) if isinstance(resource_kind, str) else None, + rows: Any = _native.read_scope_index_cache( + compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME, + fingerprint, ) + if rows is None: + return None + try: + return _index(rows) + except ValueError: + return None -def _resource_payload(item: ResourceRecord) -> JsonObject: - return { - "identity": _identity_payload(item.identity), - "path": normalize_path(path=item.path), - "ownership_root": _root_payload(item.ownership_root), - } - +def write_cached_scope_index(*, project_dir: Path, fingerprint: str, index: ScopeIndex) -> None: + """Atomically write one complete canonical scope index, ignoring I/O faults.""" -def _resource(payload: object) -> ResourceRecord: - item: JsonObject = _object(payload) - identity: ResourceIdentity | DeclarationIdentity = _identity(item.get("identity")) - if not isinstance(identity, ResourceIdentity): - raise ScopeCacheDecodeError("Invalid cached resource identity") - return ResourceRecord( - identity, - normalize_path(path=_string(value=item, key="path")), - _root(item.get("ownership_root")), + if not index.completeness.complete: + return + _ = _native.write_scope_index_cache( + (compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME), + fingerprint, + _rows(index), ) -def _declaration_payload(item: DeclarationRecord) -> JsonObject: - macro: object = None - if item.macro is not None: - macro = { - "parameters": list(item.macro.parameters), - "dependencies": [_identity_payload(value) for value in item.macro.dependencies], - "source_digest": item.macro.source_digest, - } - enum: object = None - if item.enum is not None: - enum = { - "members": [{"name": value.name} for value in item.enum.members], - "scalar_type": item.enum.scalar_type, - } - constant: object = None - if item.constant is not None: - constant = { - "logical_type": item.constant.logical_type, - "collection_kind": item.constant.collection_kind, - "item_count": item.constant.item_count, - "nullable": item.constant.nullable, - "render_as": item.constant.render_as, - } - return { - "identity": _identity_payload(item.identity), - "path": normalize_path(path=item.path), - "line": item.line, - "column": item.column, - "scope": item.scope.value, - "ownership_root": _root_payload(item.ownership_root), - "owning_path": ( - normalize_path(path=item.owning_path) if item.owning_path is not None else None +def _rows(index: ScopeIndex) -> Any: + completeness: ScopeCompleteness = index.completeness + return ( + [root_row(root=root) for root in index.ownership_roots], + [ + (_resource_row(record.identity), record.path, root_row(root=record.ownership_root)) + for record in index.resources + ], + [_declaration_row(record) for record in index.declarations], + [ + ( + qualified_row(identity=item.consumer), + qualified_row(identity=item.declaration), + item.kind.value, + _optional_row(item.through), + item.enum_member, + ) + for item in index.usages + ], + [ + ( + _resource_row(item.resource), + qualified_row(identity=item.declaration), + qualified_row(identity=item.through), + item.kind.value, + ) + for item in index.grants + ], + [ + ( + _resource_row(item.resource), + qualified_row(identity=item.declaration), + item.reason.value, + _optional_row(item.through), + ) + for item in index.visibility + ], + [ + ( + _resource_row(item.resource), + qualified_row(identity=item.declaration), + item.reason.value, + ) + for item in index.inaccessible + ], + [ + ( + item.code.value, + item.message, + item.severity.value, + item.path, + item.line, + item.column, + _optional_row(item.declaration), + None if item.resource is None else _resource_row(item.resource), + ) + for item in index.diagnostics + ], + ( + completeness.discovery, + completeness.static_visibility, + completeness.runtime_usage, + completeness.relationships, + completeness.placement, + completeness.promotion_impact, ), - "macro": macro, - "enum": enum, - "constant": constant, - } - - -def _declaration(payload: object) -> DeclarationRecord: - item: JsonObject = _object(payload) - identity: ResourceIdentity | DeclarationIdentity = _identity(item.get("identity")) - if not isinstance(identity, DeclarationIdentity): - raise ScopeCacheDecodeError("Invalid cached declaration identity") - macro_payload: object = item.get("macro") - macro: MacroMetadata | None = None - if macro_payload is not None: - value: JsonObject = _object(macro_payload) - dependencies: list[DeclarationIdentity] = [] - for raw in _list(value=value, key="dependencies"): - decoded: ResourceIdentity | DeclarationIdentity = _identity(raw) - if not isinstance(decoded, DeclarationIdentity): - raise ScopeCacheDecodeError("Invalid cached macro dependency") - dependencies.append(decoded) - macro = MacroMetadata( - parameters=tuple(_strings(value=value, key="parameters")), - dependencies=tuple(dependencies), - source_digest=_string(value=value, key="source_digest"), - ) - enum_payload: object = item.get("enum") - enum: EnumMetadata | None = None - if enum_payload is not None: - value = _object(enum_payload) - members: list[EnumMemberMetadata] = [] - for raw in _list(value=value, key="members"): - member: JsonObject = _object(raw) - if set(member) != SCOPE_ENUM_MEMBER_FIELDS: - raise ScopeCacheDecodeError("Invalid cached enum member metadata") - members.append(EnumMemberMetadata(_string(value=member, key="name"))) - enum = EnumMetadata(tuple(members), _string(value=value, key="scalar_type")) - constant_payload: object = item.get("constant") - constant: ConstantMetadata | None = None - if constant_payload is not None: - value = _object(constant_payload) - constant = ConstantMetadata( - logical_type=_string(value=value, key="logical_type"), - collection_kind=_optional_string(value.get("collection_kind")), - item_count=_optional_int(value.get("item_count")), - nullable=_bool(value=value, key="nullable"), - render_as=_optional_string(value.get("render_as")), - ) - return DeclarationRecord( - identity=identity, - path=normalize_path(path=_string(value=item, key="path")), - line=_int(value=item, key="line"), - column=_int(value=item, key="column"), - scope=ScopeKind(_string(value=item, key="scope")), - ownership_root=_root(item.get("ownership_root")), - owning_path=_optional_path(item.get("owning_path")), - macro=macro, - enum=enum, - constant=constant, ) -def _usage_payload(item: UsageRecord) -> JsonObject: - return { - "consumer": _identity_payload(item.consumer), - "declaration": _identity_payload(item.declaration), - "kind": item.kind.value, - "through": _identity_payload(item.through), - "enum_member": item.enum_member, - } - - -def _usage(payload: object) -> UsageRecord: - item: JsonObject = _object(payload) - consumer: ResourceIdentity | DeclarationIdentity = _identity(item.get("consumer")) - declaration: ResourceIdentity | DeclarationIdentity = _identity(item.get("declaration")) - through_payload: object = item.get("through") - through: ResourceIdentity | DeclarationIdentity | None = None - if through_payload is not None: - through = _identity(through_payload) - if not isinstance(declaration, DeclarationIdentity): - raise ScopeCacheDecodeError("Invalid cached usage declaration") - return UsageRecord( - consumer, - declaration, - UsageKind(_string(value=item, key="kind")), - through, - _optional_string(item.get("enum_member")), +def _declaration_row(record: DeclarationRecord) -> Any: + return ( + ( + qualified_row(identity=record.identity), + record.path, + record.line, + record.column, + record.scope.value, + root_row(root=record.ownership_root), + record.owning_path, + ), + None + if record.macro is None + else ( + list(record.macro.parameters), + [qualified_row(identity=item) for item in record.macro.dependencies], + record.macro.source_digest, + ), + None + if record.enum is None + else ([member.name for member in record.enum.members], record.enum.scalar_type), + None + if record.constant is None + else ( + record.constant.logical_type, + record.constant.collection_kind, + record.constant.item_count, + record.constant.nullable, + record.constant.render_as, + ), ) -def _grant_payload(item: GrantRecord) -> JsonObject: - return { - "resource": _identity_payload(item.resource), - "declaration": _identity_payload(item.declaration), - "through": _identity_payload(item.through), - "kind": item.kind.value, - } - - -def _grant(payload: object) -> GrantRecord: - item: JsonObject = _object(payload) - resource: ResourceIdentity | DeclarationIdentity - declaration: ResourceIdentity | DeclarationIdentity - through: ResourceIdentity | DeclarationIdentity - resource, declaration, through = ( - _identity(item.get("resource")), - _identity(item.get("declaration")), - _identity(item.get("through")), - ) - if not isinstance(resource, ResourceIdentity) or not isinstance( - through, ResourceIdentity | DeclarationIdentity - ): - raise ScopeCacheDecodeError("Invalid cached grant resource") - if not isinstance(declaration, DeclarationIdentity): - raise ScopeCacheDecodeError("Invalid cached grant declaration") - return GrantRecord(resource, declaration, through, GrantKind(_string(value=item, key="kind"))) - - -def _visibility_payload(item: VisibilityRecord) -> JsonObject: - return { - "resource": _identity_payload(item.resource), - "declaration": _identity_payload(item.declaration), - "reason": item.reason.value, - "through": _identity_payload(item.through), - } - - -def _visibility(payload: object) -> VisibilityRecord: - item: JsonObject = _object(payload) - resource: ResourceIdentity | DeclarationIdentity = _identity(item.get("resource")) - declaration: ResourceIdentity | DeclarationIdentity = _identity(item.get("declaration")) - through_payload: object = item.get("through") - through: ResourceIdentity | DeclarationIdentity | None = ( - _identity(through_payload) if through_payload is not None else None - ) - if not isinstance(resource, ResourceIdentity) or not isinstance( - declaration, DeclarationIdentity - ): - raise ScopeCacheDecodeError("Invalid cached visibility identity") - return VisibilityRecord( - resource, - declaration, - VisibilityReason(_string(value=item, key="reason")), - through, - ) +def _resource_row(identity: ResourceIdentity) -> tuple[str, str]: + return identity.kind.value, identity.name -def _inaccessible_payload(item: InaccessibleRecord) -> JsonObject: - return { - "resource": _identity_payload(item.resource), - "declaration": _identity_payload(item.declaration), - "reason": item.reason.value, - } - - -def _inaccessible(payload: object) -> InaccessibleRecord: - item: JsonObject = _object(payload) - resource: ResourceIdentity | DeclarationIdentity = _identity(item.get("resource")) - declaration: ResourceIdentity | DeclarationIdentity = _identity(item.get("declaration")) - if not isinstance(resource, ResourceIdentity) or not isinstance( - declaration, DeclarationIdentity - ): - raise ScopeCacheDecodeError("Invalid cached inaccessible identity") - return InaccessibleRecord( - resource, declaration, InaccessibleReason(_string(value=item, key="reason")) - ) +def _optional_row(identity: ResourceIdentity | DeclarationIdentity | None) -> object: + return None if identity is None else qualified_row(identity=identity) -def _diagnostic_payload(item: ScopeDiagnostic) -> JsonObject: - return { - "code": item.code.value, - "message": item.message, - "severity": item.severity.value, - "path": normalize_path(path=item.path) if item.path is not None else None, - "line": item.line, - "column": item.column, - "declaration": _identity_payload(item.declaration), - "resource": _identity_payload(item.resource), - } - - -def _diagnostic(payload: object) -> ScopeDiagnostic: - item: JsonObject = _object(payload) - declaration_payload, resource_payload = item.get("declaration"), item.get("resource") - declaration: ResourceIdentity | DeclarationIdentity | None = ( - _identity(declaration_payload) if declaration_payload is not None else None - ) - resource: ResourceIdentity | DeclarationIdentity | None = ( - _identity(resource_payload) if resource_payload is not None else None - ) - if declaration is not None and not isinstance(declaration, DeclarationIdentity): - raise ScopeCacheDecodeError("Invalid cached diagnostic declaration") - if resource is not None and not isinstance(resource, ResourceIdentity): - raise ScopeCacheDecodeError("Invalid cached diagnostic resource") - return ScopeDiagnostic( - ScopeDiagnosticCode(_string(value=item, key="code")), - _string(value=item, key="message"), - DiagnosticSeverity(_string(value=item, key="severity")), - _optional_path(item.get("path")), - _optional_int(item.get("line")), - _optional_int(item.get("column")), - declaration, - resource, +def _index(rows: Any) -> ScopeIndex: + roots, resources, declarations, usages, grants, visibility, inaccessible, diagnostics, done = ( + rows ) - - -def _completeness(payload: object) -> ScopeCompleteness: - item: JsonObject = _object(payload) - return ScopeCompleteness( - **{ - key: _bool(value=item, key=key) - for key in ( - "discovery", - "static_visibility", - "runtime_usage", - "relationships", - "placement", - "promotion_impact", + return ScopeIndex( + ownership_roots=tuple(map(_root, roots)), + resources=tuple( + ResourceRecord(_resource(identity), path, _root(root)) + for identity, path, root in resources + ), + declarations=tuple(map(_declaration, declarations)), + usages=tuple( + UsageRecord( + _identity(consumer), + _declaration_identity(declaration), + UsageKind(kind), + None if through is None else _identity(through), + enum_member, + ) + for consumer, declaration, kind, through, enum_member in usages + ), + grants=tuple( + GrantRecord( + _resource(resource), + _declaration_identity(declaration), + _identity(through), + GrantKind(kind), + ) + for resource, declaration, through, kind in grants + ), + visibility=tuple( + VisibilityRecord( + _resource(resource), + _declaration_identity(declaration), + VisibilityReason(reason), + None if through is None else _identity(through), + ) + for resource, declaration, reason, through in visibility + ), + inaccessible=tuple( + InaccessibleRecord( + _resource(resource), _declaration_identity(declaration), InaccessibleReason(reason) ) - } + for resource, declaration, reason in inaccessible + ), + diagnostics=tuple(map(_diagnostic, diagnostics)), + completeness=ScopeCompleteness(*done), ) -def _object(value: object) -> JsonObject: - if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): - raise ScopeCacheDecodeError("Expected a JSON object in scope cache") - return cast(JsonObject, value) - - -def _list(*, value: JsonObject, key: str) -> list[object]: - item: object = value.get(key) - if not isinstance(item, list): - raise ScopeCacheDecodeError(f"Expected a list for cached field '{key}'") - return cast(list[object], item) - - -def _string(*, value: JsonObject, key: str) -> str: - item: object = value.get(key) - if not isinstance(item, str): - raise ScopeCacheDecodeError(f"Expected a string for cached field '{key}'") - return item - - -def _strings(*, value: JsonObject, key: str) -> list[str]: - items: list[object] = _list(value=value, key=key) - if not all(isinstance(item, str) for item in items): - raise ScopeCacheDecodeError(f"Expected strings for cached field '{key}'") - return cast(list[str], items) - - -def _int(*, value: JsonObject, key: str) -> int: - item: object = value.get(key) - if not isinstance(item, int) or isinstance(item, bool): - raise ScopeCacheDecodeError(f"Expected an integer for cached field '{key}'") - return item - - -def _bool(*, value: JsonObject, key: str) -> bool: - item: object = value.get(key) - if not isinstance(item, bool): - raise ScopeCacheDecodeError(f"Expected a boolean for cached field '{key}'") - return item - - -def _optional_string(value: object) -> str | None: - if value is not None and not isinstance(value, str): - raise ScopeCacheDecodeError("Expected an optional string in scope cache") - return value - - -def _optional_int(value: object) -> int | None: - if value is not None and (not isinstance(value, int) or isinstance(value, bool)): - raise ScopeCacheDecodeError("Expected an optional integer in scope cache") - return value - - -def _optional_path(value: object) -> str | None: - path: str | None = _optional_string(value) - return normalize_path(path=path) if path is not None else None - - -def scope_index_fingerprint(*, project_dir: Path) -> str: - """Hash canonical scope-relevant authored inputs without absolute paths or secrets.""" - - records: list[dict[str, object]] = [] - for name in _CONFIG_FILES: - path: Path = project_dir / name - if not path.is_file(): - continue - raw: bytes = path.read_bytes() - content: bytes = _scope_config_bytes(name=name, raw=raw) - records.append(_fingerprint_record(path=name, role="config", content=content)) - roots: tuple[str, ...] = tuple( - sorted( - { - *(Path(*parts).as_posix() for parts in CANONICAL_AUTHORED_ROOTS), - *GLOBAL_DECLARATION_DIRECTORIES, - *GLOBAL_NAMED_DECLARATION_DIRECTORIES, - "python", - "seeds", - } - ) +def _declaration(row: Any) -> DeclarationRecord: + (identity, path, line, column, scope, root, owning_path), macro, enum, constant = row + return DeclarationRecord( + identity=_declaration_identity(identity), + path=path, + line=line, + column=column, + scope=ScopeKind(scope), + ownership_root=_root(root), + owning_path=owning_path, + macro=None + if macro is None + else MacroMetadata(tuple(macro[0]), tuple(map(_declaration_identity, macro[1])), macro[2]), + enum=None + if enum is None + else EnumMetadata(tuple(map(EnumMemberMetadata, enum[0])), enum[1]), + constant=None if constant is None else ConstantMetadata(*constant), ) - seen: set[str] = set() - for root in roots: - directory: Path = project_dir / root - if not directory.is_dir(): - continue - for path in sorted(item for item in directory.rglob("*") if item.is_file()): - if path.suffix.lower() not in SCOPE_SOURCE_SUFFIXES: - continue - relative: str = path.relative_to(project_dir).as_posix() - if relative in seen: - continue - seen.add(relative) - records.append( - _fingerprint_record( - path=relative, - role=_source_role(path=Path(relative)), - content=path.read_bytes(), - ) - ) - payload: dict[str, object] = { - "algorithm_version": SCOPE_FINGERPRINT_ALGORITHM_VERSION, - "cache_schema_version": SCOPE_CACHE_SCHEMA_VERSION, - "ownership_roots": list(roots), - "sources": sorted(records, key=lambda item: (str(item["path"]), str(item["role"]))), - "sqlbuild_version": _sqlbuild_version(), - } - encoded: bytes = _canonical_json(payload) - return hashlib.sha256(encoded).hexdigest() - -def read_cached_scope_index(*, project_dir: Path, fingerprint: str) -> ScopeIndex | None: - """Read a verified index; every storage or validation fault is a cache miss.""" - path: Path = ( - compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME +def _diagnostic(row: Any) -> ScopeDiagnostic: + code, message, severity, path, line, column, declaration, resource = row + return ScopeDiagnostic( + ScopeDiagnosticCode(code), + message, + DiagnosticSeverity(severity), + path, + line, + column, + None if declaration is None else _declaration_identity(declaration), + None if resource is None else _resource(resource), ) - try: - if path.stat().st_size > SCOPE_CACHE_MAX_BYTES: - return None - raw: bytes = path.read_bytes() - envelope: object = json.loads(raw.decode("utf-8")) - if not isinstance(envelope, dict): - return None - if envelope.get("schema_version") != SCOPE_CACHE_SCHEMA_VERSION: - return None - if envelope.get("input_fingerprint") != fingerprint: - return None - payload: object = envelope.get("payload") - payload_bytes: bytes = _canonical_json(payload) - if envelope.get("payload_sha256") != hashlib.sha256(payload_bytes).hexdigest(): - return None - return decode_scope_index(payload=payload) - except (OSError, UnicodeError, json.JSONDecodeError, TypeError, ValueError): - return None - -def write_cached_scope_index(*, project_dir: Path, fingerprint: str, index: ScopeIndex) -> None: - """Atomically write one complete canonical scope index, ignoring I/O faults.""" - if not index.completeness.complete: - return - try: - payload: object = json.loads(encode_scope_index(index=index).decode("ascii")) - payload_bytes: bytes = _canonical_json(payload) - envelope: bytes = _canonical_json( - { - "input_fingerprint": fingerprint, - "payload": payload, - "payload_sha256": hashlib.sha256(payload_bytes).hexdigest(), - "schema_version": SCOPE_CACHE_SCHEMA_VERSION, - } - ) - directory: Path = compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME - directory.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp(prefix=".scope-index-", dir=directory) - temporary_path: Path = Path(temporary_name) - try: - with os.fdopen(descriptor, "wb") as stream: - stream.write(envelope) - stream.flush() - os.fsync(stream.fileno()) - temporary_path.replace(directory / SCOPE_CACHE_FILENAME) - finally: - temporary_path.unlink(missing_ok=True) - except (OSError, TypeError, ValueError): - return +def _root(row: tuple[str, str, str | None]) -> OwnershipRoot: + path, kind, resource_kind = row + return OwnershipRoot( + path, + OwnershipRootKind(kind), + None if resource_kind is None else ResourceKind(resource_kind), + ) -def _fingerprint_record(*, path: str, role: str, content: bytes) -> dict[str, object]: - return { - "path": path.replace("\\", "/"), - "role": role, - "sha256": hashlib.sha256(content).hexdigest(), - } +def _resource(row: tuple[str, str]) -> ResourceIdentity: + return ResourceIdentity(ResourceKind(row[0]), row[1]) -def _source_role(*, path: Path) -> str: - parts: tuple[str, ...] = path.parts - if any(part in GLOBAL_DECLARATION_DIRECTORIES for part in parts) or ( - parts[:1] and parts[0] in GLOBAL_NAMED_DECLARATION_DIRECTORIES - ): - return "declaration" - if path.suffix == SCOPE_MACRO_SUFFIX: - return "macro" - if parts[:2] in SCOPE_RELATIONSHIP_ROOTS: - return "relationship" - return "resource" +def _declaration_identity(row: NativeQualifiedRow) -> DeclarationIdentity: + kind, name, owner = row + return DeclarationIdentity( + DeclarationKind(kind), name, None if owner is None else _resource(owner) + ) -def _scope_config_bytes(*, name: str, raw: bytes) -> bytes: - if not name.endswith(".toml"): - return raw - try: - payload: object = tomllib.loads(raw.decode("utf-8")) - except (UnicodeError, tomllib.TOMLDecodeError): - return raw - if not isinstance(payload, dict): - return raw - relevant: dict[str, object] - if name == LOCAL_CONFIG_FILENAME: - relevant = {key: value for key, value in payload.items() if key in SCOPE_LOCAL_CONFIG_KEYS} - else: - relevant = { - key: value for key, value in payload.items() if key in SCOPE_PROJECT_CONFIG_KEYS - } - if SCOPE_TARGETS_KEY in relevant: - relevant[SCOPE_TARGETS_KEY] = _scope_target_configs(value=relevant[SCOPE_TARGETS_KEY]) - return _canonical_json(relevant) - - -def _scope_target_configs(*, value: object) -> dict[str, object]: - if not isinstance(value, dict): - return {} - targets: dict[str, object] = {} - for raw_name, raw_config in value.items(): - if not isinstance(raw_name, str) or not isinstance(raw_config, dict): - continue - config: dict[str, object] = cast(dict[str, object], raw_config) - targets[raw_name] = { - SCOPE_TARGET_VARS_KEY: config.get(SCOPE_TARGET_VARS_KEY, {}), - } - return targets +def _identity(row: NativeQualifiedRow) -> ResourceIdentity | DeclarationIdentity: + if row[2] is None and row[0] in _RESOURCE_KINDS: + return _resource((row[0], row[1])) + return _declaration_identity(row) def _sqlbuild_version() -> str: @@ -692,9 +335,3 @@ def _sqlbuild_version() -> str: return version("sqlbuild") except PackageNotFoundError: return "0+unknown" - - -def _canonical_json(payload: object) -> bytes: - return json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True).encode( - "ascii" - ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/identities.py b/src/sqlbuild/compiler/scopes/_helpers/identities.py deleted file mode 100644 index 618b42fb82..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/identities.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Pure qualified scope identity implementation.""" - -from __future__ import annotations - -from sqlbuild.compiler.scopes.constants import ( - PRIVATE_IDENTITY_PART_COUNT, - PUBLIC_IDENTITY_PART_COUNT, -) -from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind - - -def format_identity(*, identity: ResourceIdentity | DeclarationIdentity) -> str: - if isinstance(identity, ResourceIdentity): - return f"{identity.kind.value}:{identity.name}" - if identity.owner is None: - return f"{identity.kind.value}:{identity.name}" - return ( - f"{identity.kind.value}:{identity.owner.kind.value}:{identity.owner.name}.{identity.name}" - ) - - -def parse_identity(*, value: str) -> ResourceIdentity | DeclarationIdentity: - parts: list[str] = value.split(":") - if len(parts) == PUBLIC_IDENTITY_PART_COUNT: - raw_kind, name = parts - if not name: - raise InvalidQualifiedIdentityError(f"Qualified identity has no name: {value!r}") - try: - return ResourceIdentity(kind=ResourceKind(raw_kind), name=name) - except ValueError: - try: - return DeclarationIdentity(kind=DeclarationKind(raw_kind), name=name) - except ValueError as error: - raise InvalidQualifiedIdentityError( - f"Unknown qualified identity kind: {raw_kind!r}" - ) from error - if len(parts) == PRIVATE_IDENTITY_PART_COUNT: - raw_kind, raw_owner_kind, owner_and_name = parts - try: - kind: DeclarationKind = DeclarationKind(raw_kind) - owner_kind: ResourceKind = ResourceKind(raw_owner_kind) - except ValueError as error: - raise InvalidQualifiedIdentityError(f"Invalid private identity: {value!r}") from error - owner_name, separator, name = owner_and_name.rpartition(".") - if not separator or not owner_name or not name: - raise InvalidQualifiedIdentityError(f"Invalid private identity: {value!r}") - return DeclarationIdentity( - kind=kind, - name=name, - owner=ResourceIdentity(kind=owner_kind, name=owner_name), - ) - raise InvalidQualifiedIdentityError( - f"Expected a kind-qualified identity such as 'model:name': {value!r}" - ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/lookup.py b/src/sqlbuild/compiler/scopes/_helpers/lookup.py index 4265621753..04bb489b58 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/lookup.py +++ b/src/sqlbuild/compiler/scopes/_helpers/lookup.py @@ -1,174 +1,186 @@ -"""Canonical scope lookup construction implementation.""" +"""Canonical scope lookups, ordered and grouped by the native scope lookup.""" from __future__ import annotations -from collections import defaultdict -from collections.abc import Callable, Hashable, Iterable -from functools import lru_cache +from collections.abc import Callable, Hashable, Sequence from types import MappingProxyType -from sqlbuild.compiler.scopes._helpers.identities import format_identity -from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes._helpers.visibility import build_visibility_index +import sqlbuild._native as _native +from sqlbuild.compiler.scopes._helpers.placement import qualified_row, root_row +from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, + DeclarationVisibilityIndex, GrantRecord, - InaccessibleRecord, ResourceIdentity, ResourceRecord, ScopeIndex, ScopeLookup, UsageRecord, - VisibilityRecord, ) +from sqlbuild.compiler.scopes.types import ResourceKind - -def group_records[Record, Key: Hashable]( - *, - records: Iterable[Record], - key: Callable[[Record], Key], - sort_key: Callable[[Record], tuple[str, ...]], -) -> MappingProxyType[Key, tuple[Record, ...]]: - grouped: dict[Key, list[Record]] = defaultdict(list) - for record in records: - grouped[key(record)].append(record) - return MappingProxyType( - { - group_key: tuple(sorted(values, key=sort_key)) - for group_key, values in sorted(grouped.items(), key=lambda item: repr(item[0])) - } - ) +type _Groups = list[list[int]] +type _Mappings = tuple[_Groups, _Groups, _Groups, _Groups, _Groups, _Groups, list[str]] +type _Visibility = tuple[ + list[int], + list[tuple[tuple[str, str], list[int]]], + list[tuple[str, list[int]]], + list[tuple[str, list[int]]], +] def build_lookup(*, index: ScopeIndex) -> ScopeLookup: - resources: list[ResourceRecord] = sorted(index.resources, key=resource_sort_key) - declarations: list[DeclarationRecord] = sorted(index.declarations, key=declaration_sort_key) - resource_map: MappingProxyType[ResourceIdentity, tuple[ResourceRecord, ...]] = group_records( - records=resources, - key=lambda item: item.identity, - sort_key=resource_sort_key, - ) - declaration_map: MappingProxyType[DeclarationIdentity, tuple[DeclarationRecord, ...]] = ( - group_records( - records=declarations, - key=lambda item: item.identity, - sort_key=declaration_sort_key, + """The native lookup of `index`; a path the lookup rejects raises `InvalidScopePathError`.""" + + native: _native.NativeScopeLookup | None + native, (rows, error) = _native.build_native_scope_lookup( + ( + [root_row(root=root) for root in index.ownership_roots], + [ + ( + (record.identity.kind.value, record.identity.name), + record.path, + root_row(root=record.ownership_root), + ) + for record in index.resources + ], + [ + ( + qualified_row(identity=record.identity), + record.path, + record.line, + record.column, + record.scope.value, + root_row(root=record.ownership_root), + record.owning_path, + ) + for record in index.declarations + ], + [ + ( + qualified_row(identity=usage.consumer), + qualified_row(identity=usage.declaration), + usage.kind.value, + None if usage.through is None else qualified_row(identity=usage.through), + ) + for usage in index.usages + ], + [ + ( + (grant.resource.kind.value, grant.resource.name), + qualified_row(identity=grant.declaration), + qualified_row(identity=grant.through), + grant.kind.value, + ) + for grant in index.grants + ], + [(item.path, item.line, item.column, item.code.value) for item in index.diagnostics], ) ) - - paths: list[tuple[str, ResourceRecord]] = [] - for record in resources: - path: str = normalize_path(path=record.path) - paths.append((path, record)) - path_map: MappingProxyType[str, tuple[tuple[str, ResourceRecord], ...]] = group_records( - records=paths, - key=lambda item: item[0], - sort_key=lambda item: resource_sort_key(item[1]), - ) - - canonical_index: ScopeIndex = ScopeIndex( - ownership_roots=tuple( - sorted( - index.ownership_roots, - key=lambda item: ( - item.path, - item.resource_kind.value if item.resource_kind is not None else "", - ), - ) - ), - resources=tuple(resources), - declarations=tuple(declarations), - usages=tuple(sorted(index.usages, key=usage_sort_key)), - grants=tuple(sorted(index.grants, key=grant_sort_key)), - visibility=tuple(sorted(index.visibility, key=visibility_sort_key)), - inaccessible=tuple(sorted(index.inaccessible, key=inaccessible_sort_key)), - diagnostics=tuple( - sorted( - index.diagnostics, - key=lambda item: ( - item.path or "", - item.line or 0, - item.column or 0, - item.code.value, - ), - ) - ), + if rows is None: + raise InvalidScopePathError(error or "") + (roots, resources, declarations, usages, grants, diagnostics), mappings, visibility = rows + canonical: ScopeIndex = ScopeIndex( + ownership_roots=tuple(index.ownership_roots[position] for position in roots), + resources=tuple(index.resources[position] for position in resources), + declarations=tuple(index.declarations[position] for position in declarations), + usages=tuple(index.usages[position] for position in usages), + grants=tuple(index.grants[position] for position in grants), + visibility=index.visibility, + inaccessible=index.inaccessible, + diagnostics=tuple(index.diagnostics[position] for position in diagnostics), completeness=index.completeness, ) - resources_by_path: dict[str, tuple[ResourceRecord, ...]] = {} - for path, records_for_path in path_map.items(): - resources_by_path[path] = tuple(item[1] for item in records_for_path) + return project_lookup( + index=canonical, + records=(index.resources, index.declarations, index.usages, index.grants), + mappings=mappings, + visibility=visibility, + native=native, + ) + + +def project_lookup( + *, + index: ScopeIndex, + records: tuple[ + Sequence[ResourceRecord], + Sequence[DeclarationRecord], + Sequence[UsageRecord], + Sequence[GrantRecord], + ], + mappings: _Mappings, + visibility: _Visibility, + native: object | None, +) -> ScopeLookup: + """Project native lookup groups over `records` as a `ScopeLookup` of canonical `index`.""" + + resources, declarations, usages, grants = records + global_positions, private_positions, local_positions, inherited_positions = visibility return ScopeLookup( - index=canonical_index, - resources=resource_map, - resources_by_path=MappingProxyType(resources_by_path), - declarations=declaration_map, - usages_by_consumer=group_records( - records=canonical_index.usages, key=lambda item: item.consumer, sort_key=usage_sort_key + index=index, + resources=_mapping(groups=mappings[0], records=resources, key=_resource_identity), + resources_by_path=_path_mapping(groups=mappings[1], records=resources, paths=mappings[6]), + declarations=_mapping(groups=mappings[2], records=declarations, key=_declaration_identity), + usages_by_consumer=_mapping(groups=mappings[3], records=usages, key=_usage_consumer), + usages_by_declaration=_mapping(groups=mappings[4], records=usages, key=_usage_declaration), + grants_by_resource=_mapping(groups=mappings[5], records=grants, key=_grant_resource), + visibility_index=DeclarationVisibilityIndex( + identities=tuple(record.identity for record in index.declarations), + global_positions=tuple(global_positions), + private_positions=MappingProxyType( + { + ResourceIdentity(ResourceKind(kind), name): tuple(positions) + for (kind, name), positions in private_positions + } + ), + local_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in local_positions} + ), + inherited_positions=MappingProxyType( + {owner: tuple(positions) for owner, positions in inherited_positions} + ), ), - usages_by_declaration=group_records( - records=canonical_index.usages, - key=lambda item: item.declaration, - sort_key=usage_sort_key, - ), - grants_by_resource=group_records( - records=canonical_index.grants, - key=lambda item: item.resource, - sort_key=grant_sort_key, - ), - visibility_index=build_visibility_index(declarations=canonical_index.declarations), + native=native, ) -@lru_cache(maxsize=65_536) -def identity_key(identity: ResourceIdentity | DeclarationIdentity) -> str: - return format_identity(identity=identity) +def _resource_identity(record: ResourceRecord) -> ResourceIdentity: + return record.identity -def resource_sort_key(record: ResourceRecord) -> tuple[str, ...]: - return (identity_key(record.identity), normalize_path(path=record.path)) +def _declaration_identity(record: DeclarationRecord) -> DeclarationIdentity: + return record.identity -def declaration_sort_key(record: DeclarationRecord) -> tuple[str, ...]: - return ( - identity_key(record.identity), - normalize_path(path=record.path), - str(record.line), - str(record.column), - ) +def _usage_consumer(record: UsageRecord) -> ResourceIdentity | DeclarationIdentity: + return record.consumer -def usage_sort_key(record: UsageRecord) -> tuple[str, ...]: - return ( - identity_key(record.consumer), - identity_key(record.declaration), - record.kind.value, - identity_key(record.through) if record.through else "", - ) +def _usage_declaration(record: UsageRecord) -> DeclarationIdentity: + return record.declaration -def grant_sort_key(record: GrantRecord) -> tuple[str, ...]: - return ( - identity_key(record.resource), - identity_key(record.declaration), - identity_key(record.through), - record.kind.value, - ) +def _grant_resource(record: GrantRecord) -> ResourceIdentity: + return record.resource -def visibility_sort_key(record: VisibilityRecord) -> tuple[str, ...]: - return ( - identity_key(record.resource), - identity_key(record.declaration), - record.reason.value, - identity_key(record.through) if record.through else "", - ) +def _path_mapping( + *, groups: _Groups, records: Sequence[ResourceRecord], paths: list[str] +) -> MappingProxyType[str, tuple[ResourceRecord, ...]]: + items: list[tuple[str, tuple[ResourceRecord, ...]]] = [] + for group in groups: + items.append((paths[group[0]], tuple(records[member] for member in group))) + return MappingProxyType(dict(items)) -def inaccessible_sort_key(record: InaccessibleRecord) -> tuple[str, ...]: - return ( - identity_key(record.resource), - identity_key(record.declaration), - record.reason.value, - ) +def _mapping[Record, Key: Hashable]( + *, groups: _Groups, records: Sequence[Record], key: Callable[[Record], Key] +) -> MappingProxyType[Key, tuple[Record, ...]]: + items: list[tuple[Key, tuple[Record, ...]]] = [] + for group in groups: + grouped: tuple[Record, ...] = tuple(records[member] for member in group) + items.append((key(grouped[0]), grouped)) + return MappingProxyType(dict(items)) diff --git a/src/sqlbuild/compiler/scopes/_helpers/paths.py b/src/sqlbuild/compiler/scopes/_helpers/paths.py index 3b08e5ea66..b63d0b00e8 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/paths.py +++ b/src/sqlbuild/compiler/scopes/_helpers/paths.py @@ -1,51 +1,17 @@ -"""Project-relative path normalization implementation.""" +"""Project-relative scope path normalization, answered by the native scope paths.""" from __future__ import annotations -from functools import lru_cache from pathlib import PurePath -from sqlbuild.compiler.scopes.constants import ( - CURRENT_PATH_COMPONENT, - PARENT_PATH_COMPONENT, - PATH_SEPARATOR, - WINDOWS_DRIVE_PREFIX_LENGTH, - WINDOWS_DRIVE_SEPARATOR, -) +import sqlbuild._native as _native from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError -@lru_cache(maxsize=65_536) def normalize_path(*, path: str | PurePath) -> str: - display_path: str = str(path) - raw: str = display_path.replace("\\", PATH_SEPARATOR) - has_drive: bool = len(raw) >= WINDOWS_DRIVE_PREFIX_LENGTH and raw[1] == WINDOWS_DRIVE_SEPARATOR - if not raw or raw.startswith(PATH_SEPARATOR) or has_drive: - raise InvalidScopePathError(f"Scope path must be project-relative: {display_path}") + """The canonical project-relative POSIX path; absolute or escaping paths raise.""" - components: list[str] = [] - ignored_components: frozenset[str] = frozenset({"", CURRENT_PATH_COMPONENT}) - for component in raw.split(PATH_SEPARATOR): - if component in ignored_components: - continue - if component == PARENT_PATH_COMPONENT: - if not components: - raise InvalidScopePathError(f"Scope path escapes the project: {display_path}") - components.pop() - continue - components.append(component) - return PATH_SEPARATOR.join(components) or CURRENT_PATH_COMPONENT - - -def is_equal_or_descendant(*, path: str | PurePath, ancestor: str | PurePath) -> bool: - path_parts: tuple[str, ...] = _path_parts(normalize_path(path=path)) - ancestor_path: str = normalize_path(path=ancestor) - ancestor_parts: tuple[str, ...] = ( - () if ancestor_path == CURRENT_PATH_COMPONENT else _path_parts(ancestor_path) - ) - return path_parts[: len(ancestor_parts)] == ancestor_parts - - -@lru_cache(maxsize=65_536) -def _path_parts(path: str) -> tuple[str, ...]: - return tuple(path.split(PATH_SEPARATOR)) + normalized, error = _native.scope_normalize_path(str(path)) + if normalized is None: + raise InvalidScopePathError(error or "") + return normalized diff --git a/src/sqlbuild/compiler/scopes/_helpers/placement.py b/src/sqlbuild/compiler/scopes/_helpers/placement.py index d2d1296274..bf749ea471 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/placement.py +++ b/src/sqlbuild/compiler/scopes/_helpers/placement.py @@ -1,24 +1,15 @@ -"""Validate declaration usage and exact lexical placement.""" +"""Validate declaration usage and exact lexical placement with the native placement rules.""" from __future__ import annotations -from collections import deque from dataclasses import replace -from pathlib import PurePosixPath -from sqlbuild.compiler.discovery.constants import CANONICAL_AUTHORED_ROOTS -from sqlbuild.compiler.scopes._helpers.identities import format_identity -from sqlbuild.compiler.scopes.constants import ( - DECLARATION_GROUP_DIRECTORY, - DECLARATION_ROLE_PARTS, - NAMED_DECLARATION_KINDS, -) +import sqlbuild._native as _native from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, OwnershipRoot, ResourceIdentity, - ResourceRecord, ScopeDiagnostic, ScopeIndex, UsageRecord, @@ -26,15 +17,16 @@ from sqlbuild.compiler.scopes.types import ( DeclarationKind, DiagnosticSeverity, + NativePlacementRows, + NativeQualifiedRow, + NativeRootRow, + NativeUsageKeyRow, OwnershipRootKind, + ResourceKind, ScopeDiagnosticCode, ScopeKind, ) -type _Anchor = tuple[OwnershipRoot, str] -type _UsagesByDeclaration = dict[DeclarationIdentity, tuple[UsageRecord, ...]] -type _AnchorSets = dict[DeclarationIdentity, frozenset[_Anchor]] - _PLACEMENT_CODES: frozenset[ScopeDiagnosticCode] = frozenset( { ScopeDiagnosticCode.UNUSED_DECLARATION, @@ -44,10 +36,70 @@ ScopeDiagnosticCode.OVER_BROAD_GLOBAL, } ) -_PROJECT_ROOT_PATH: str = "." -_AUTHORED_ROOT_PATHS: frozenset[str] = frozenset( - PurePosixPath(*parts).as_posix() for parts in CANONICAL_AUTHORED_ROOTS -) + + +def merged_usage_index(*, index: ScopeIndex, collected: tuple[UsageRecord, ...]) -> ScopeIndex: + """The index with compile usages merged into its usages and macro dependencies.""" + + distinct, merged, combined = _native.scope_merged_usages( + [_dependency_row(record=record) for record in index.declarations], + [_usage_row(usage=usage) for usage in index.usages], + [_usage_row(usage=usage) for usage in collected], + ) + usages: list[UsageRecord] = [collected[position] for position in distinct] + all_usages: list[UsageRecord] = [*index.usages, *usages] + return replace( + index, + declarations=tuple( + _with_dependencies(record=record, sources=sources, usages=usages) + for record, sources in zip(index.declarations, merged, strict=True) + ), + usages=tuple(all_usages[position] for position in combined), + completeness=replace(index.completeness, runtime_usage=True, promotion_impact=True), + ) + + +def _dependency_row( + *, record: DeclarationRecord +) -> tuple[NativeQualifiedRow, list[NativeQualifiedRow] | None]: + dependencies: tuple[DeclarationIdentity, ...] | None = ( + None if record.macro is None else record.macro.dependencies + ) + return ( + qualified_row(identity=record.identity), + None if dependencies is None else [qualified_row(identity=item) for item in dependencies], + ) + + +def _with_dependencies( + *, + record: DeclarationRecord, + sources: list[tuple[bool, int]] | None, + usages: list[UsageRecord], +) -> DeclarationRecord: + if record.macro is None or sources is None: + return record + own: tuple[DeclarationIdentity, ...] = record.macro.dependencies + return replace( + record, + macro=replace( + record.macro, + dependencies=tuple( + own[position] if is_own else usages[position].declaration + for is_own, position in sources + ), + ), + ) + + +def _usage_row(*, usage: UsageRecord) -> NativeUsageKeyRow: + return ( + qualified_row(identity=usage.consumer), + qualified_row(identity=usage.declaration), + usage.kind.value, + None if usage.through is None else qualified_row(identity=usage.through), + usage.enum_member, + ) def build_placement_validated_index( @@ -55,134 +107,42 @@ def build_placement_validated_index( ) -> ScopeIndex: """Return the index with deterministic unused and exact-placement diagnostics.""" + kept: list[ScopeDiagnostic] = [ + item for item in index.diagnostics if item.code not in _PLACEMENT_CODES + ] if not index.completeness.runtime_usage: return replace( index, - diagnostics=tuple( - item for item in index.diagnostics if item.code not in _PLACEMENT_CODES - ), + diagnostics=tuple(kept), completeness=replace(index.completeness, placement=False), ) - usages_by_declaration: _UsagesByDeclaration = _usages_by_declaration(index=index) - anchor_sets: _AnchorSets = _anchor_sets( - index=index, usages_by_declaration=usages_by_declaration - ) - - diagnostics: list[ScopeDiagnostic] = [ - item for item in index.diagnostics if item.code not in _PLACEMENT_CODES - ] - for declaration in index.declarations: - declaration_usages: tuple[UsageRecord, ...] = usages_by_declaration.get( - declaration.identity, () - ) - if declaration.scope is ScopeKind.PRIVATE: - declaration_usages = tuple( - usage - for usage in declaration_usages - if usage.consumer == declaration.identity.owner and usage.through is None - ) - if not declaration_usages: - if declaration.identity.kind is DeclarationKind.SINGULAR_AUDIT: - continue - diagnostics.append( - _diagnostic( - declaration=declaration, - code=ScopeDiagnosticCode.UNUSED_DECLARATION, - enforce_placement=enforce_placement, - message=f"Unused {declaration.scope.value} declaration " - f"'{format_identity(identity=declaration.identity)}' at {declaration.path}; " - "remove it or add a genuine runtime use", - ) - ) - continue - if declaration.scope is ScopeKind.PRIVATE: - continue - required: tuple[ScopeKind, str | None, tuple[str, ...]] | None = ( - _required_placement_for_record( - record=declaration, - usages_by_declaration=usages_by_declaration, - anchor_sets=anchor_sets, - ) - ) - if required is None: - continue - required_scope, required_path, consumer_labels = required - consumers: str = ", ".join(consumer_labels) - if declaration.scope is required_scope and declaration.owning_path == required_path: - continue - if required_scope is ScopeKind.GLOBAL: - diagnostics.append( - _diagnostic( - declaration=declaration, - code=ScopeDiagnosticCode.REQUIRES_GLOBAL_PLACEMENT, - enforce_placement=enforce_placement, - message=_message( - declaration=declaration, - required_scope=ScopeKind.GLOBAL, - required_path=None, - consumers=consumers, - ), - ) - ) - continue - code: ScopeDiagnosticCode = ( - ScopeDiagnosticCode.LOCAL_NEEDED_BY_DESCENDANT - if declaration.scope is ScopeKind.LOCAL - else ( - ScopeDiagnosticCode.OVER_BROAD_GLOBAL - if declaration.scope is ScopeKind.GLOBAL - else ScopeDiagnosticCode.OVER_BROAD_INHERITED - ) - ) - diagnostics.append( - _diagnostic( - declaration=declaration, - code=code, - enforce_placement=enforce_placement, - message=_message( - declaration=declaration, - required_scope=required_scope, - required_path=required_path, - consumers=consumers, - ), + severity: DiagnosticSeverity = ( + DiagnosticSeverity.ERROR if enforce_placement else DiagnosticSeverity.WARNING + ) + for position, code, message in _native.scope_placement_diagnostics(_rows(index=index)): + declaration: DeclarationRecord = index.declarations[position] + kept.append( + ScopeDiagnostic( + code=ScopeDiagnosticCode(code), + message=message, + path=declaration.path, + line=declaration.line, + column=declaration.column, + declaration=declaration.identity, + severity=severity, ) ) return replace( index, - diagnostics=tuple(sorted(diagnostics, key=_diagnostic_key)), + diagnostics=tuple(sorted(kept, key=_diagnostic_key)), completeness=replace(index.completeness, placement=True), ) -def resolve_required_placement( - *, index: ScopeIndex, declaration: DeclarationIdentity -) -> tuple[ScopeKind, str | None, tuple[str, ...]] | None: - """Return exact required placement using the validation anchor semantics.""" - - records: tuple[DeclarationRecord, ...] = tuple( - item for item in index.declarations if item.identity == declaration - ) - if not records or not index.completeness.runtime_usage: - return None - usages_by_declaration: _UsagesByDeclaration = _usages_by_declaration(index=index) - return _required_placement_for_record( - record=records[0], - usages_by_declaration=usages_by_declaration, - anchor_sets=_anchor_sets(index=index, usages_by_declaration=usages_by_declaration), - ) - - def declaration_directory(*, kind: DeclarationKind, scope: ScopeKind, path: str | None) -> str: """Return the project-relative folder where a declaration with this placement lives.""" - role_parts: tuple[str, ...] = DECLARATION_ROLE_PARTS[kind] - if scope is ScopeKind.GLOBAL or path is None: - return "/".join(role_parts) - prefix: str = "_" if scope is ScopeKind.LOCAL else "" - role: str = "/".join((f"{prefix}{role_parts[0]}", *role_parts[1:])) - if path in _AUTHORED_ROOT_PATHS: - return f"{path}/{role}" - return f"{path}/{DECLARATION_GROUP_DIRECTORY}/{role}" + return _native.scope_declaration_directory(kind.value, scope.value, path) def relocate_for_resource_move( @@ -192,270 +152,79 @@ def relocate_for_resource_move( if not index.completeness.runtime_usage: return None - misplaced: frozenset[DeclarationIdentity] = frozenset( - item.declaration - for item in build_placement_validated_index(index=index).diagnostics - if item.code in _PLACEMENT_CODES and item.declaration is not None - ) - current: ScopeIndex = replace( - index, - resources=tuple( - replace(item, path=destination) if item.identity == resource else item - for item in index.resources - ), - ) - usages_by_declaration: _UsagesByDeclaration = _usages_by_declaration(index=current) - for _ in range(len(index.declarations) + 1): - anchor_sets: _AnchorSets = _anchor_sets( - index=current, usages_by_declaration=usages_by_declaration + relocations: list[tuple[int, str, str, NativeRootRow, str | None]] | None = ( + _native.scope_relocations( + _rows(index=index), (resource.kind.value, resource.name), destination ) - relocated: dict[DeclarationIdentity, DeclarationRecord] = {} - record: DeclarationRecord - for record in current.declarations: - if record.scope is ScopeKind.PRIVATE or record.identity in misplaced: - continue - moved: DeclarationRecord | None = _relocated_record( - record=record, - usages_by_declaration=usages_by_declaration, - anchor_sets=anchor_sets, - ) - if moved is not None: - relocated[record.identity] = moved - if not relocated: - original: dict[DeclarationIdentity, DeclarationRecord] = { - item.identity: item for item in index.declarations - } - return tuple( - item for item in current.declarations if item.path != original[item.identity].path - ) - current = replace( - current, - declarations=tuple(relocated.get(item.identity, item) for item in current.declarations), - ) - return None - - -def _relocated_record( - *, - record: DeclarationRecord, - usages_by_declaration: _UsagesByDeclaration, - anchor_sets: _AnchorSets, -) -> DeclarationRecord | None: - required: tuple[ScopeKind, str | None, tuple[str, ...]] | None = _required_placement_for_record( - record=record, usages_by_declaration=usages_by_declaration, anchor_sets=anchor_sets ) - if required is None: + if relocations is None: return None - scope: ScopeKind = required[0] - path: str | None = required[1] - if record.scope is scope and record.owning_path == path: - return None - directory: str = declaration_directory(kind=record.identity.kind, scope=scope, path=path) - roots: set[OwnershipRoot] = { - root for root, _path in anchor_sets.get(record.identity, frozenset()) - } - ownership_root: OwnershipRoot = ( - OwnershipRoot(directory, OwnershipRootKind.GLOBAL) - if scope is ScopeKind.GLOBAL or len(roots) != 1 - else next(iter(roots)) - ) - return replace( - record, - path=f"{directory}/{PurePosixPath(record.path).name}", - scope=scope, - ownership_root=ownership_root, - owning_path=None if scope is ScopeKind.GLOBAL else path, - ) - - -def _usages_by_declaration(*, index: ScopeIndex) -> _UsagesByDeclaration: - usage_lists: dict[DeclarationIdentity, list[UsageRecord]] = {} - for usage in index.usages: - usage_lists.setdefault(usage.declaration, []).append(usage) - return {identity: tuple(usages) for identity, usages in usage_lists.items()} - - -def _required_placement_for_record( - *, - record: DeclarationRecord, - usages_by_declaration: _UsagesByDeclaration, - anchor_sets: _AnchorSets, -) -> tuple[ScopeKind, str | None, tuple[str, ...]] | None: - declaration: DeclarationIdentity = record.identity - declaration_usages: tuple[UsageRecord, ...] = usages_by_declaration.get(declaration, ()) - if record.scope is ScopeKind.PRIVATE: - direct_private: tuple[UsageRecord, ...] = tuple( - usage - for usage in declaration_usages - if usage.consumer == declaration.owner and usage.through is None + return tuple( + replace( + index.declarations[position], + path=path, + scope=ScopeKind(scope), + ownership_root=OwnershipRoot( + root_path, + OwnershipRootKind(root_kind), + None if resource_kind is None else ResourceKind(resource_kind), + ), + owning_path=owning_path, ) - return ( - (ScopeKind.PRIVATE, record.owning_path or record.ownership_root.path, ()) - if direct_private - else None + for position, path, scope, (root_path, root_kind, resource_kind), owning_path in ( + relocations ) - if not declaration_usages: - return None - anchors: frozenset[_Anchor] = anchor_sets.get(declaration, frozenset()) - if not anchors: - return None - consumers: tuple[str, ...] = tuple( - sorted({_consumer_label(usage) for usage in declaration_usages}) ) - roots: set[OwnershipRoot] = {root for root, _path in anchors} - if len(roots) != 1 or any(root.kind is OwnershipRootKind.GLOBAL for root in roots): - return ScopeKind.GLOBAL, None, consumers - ownership_root: OwnershipRoot = next(iter(roots)) - paths: tuple[str, ...] = tuple(path for _root, path in anchors) - distinct: set[str] = set(paths) - named: bool = declaration.kind in NAMED_DECLARATION_KINDS - if len(distinct) == 1: - required_path: str = next(iter(distinct)) - if named and required_path == ownership_root.path: - return ScopeKind.GLOBAL, None, consumers - if declaration.kind is DeclarationKind.SINGULAR_AUDIT: - return ScopeKind.INHERITED, required_path, consumers - return ScopeKind.LOCAL, required_path, consumers - required_path = _lca(paths) - return ( - (ScopeKind.GLOBAL, None, consumers) - if required_path == ownership_root.path - else (ScopeKind.INHERITED, required_path, consumers) - ) - - -def _anchor_sets(*, index: ScopeIndex, usages_by_declaration: _UsagesByDeclaration) -> _AnchorSets: - """Return each declaration's resource anchors reachable through consumer chains.""" - - resources: dict[ResourceIdentity, ResourceRecord] = { - item.identity: item for item in index.resources - } - declarations: dict[DeclarationIdentity, DeclarationRecord] = { - item.identity: item for item in index.declarations - } - anchors: dict[DeclarationIdentity, set[_Anchor]] = {} - dependents: dict[DeclarationIdentity, set[DeclarationIdentity]] = {} - for identity, declaration_usages in usages_by_declaration.items(): - direct: set[_Anchor] = anchors.setdefault(identity, set()) - for usage in declaration_usages: - if isinstance(usage.through, DeclarationIdentity): - through_declaration: DeclarationRecord | None = declarations.get(usage.through) - if through_declaration is not None and through_declaration.owning_path is not None: - direct.add( - ( - through_declaration.ownership_root, - through_declaration.owning_path, - ) - ) - continue - if ( - usage.through is None - and isinstance(usage.consumer, DeclarationIdentity) - and usage.consumer.kind in NAMED_DECLARATION_KINDS - ): - consumer_declaration: DeclarationRecord | None = declarations.get(usage.consumer) - if consumer_declaration is not None: - direct.add(_declaration_location_anchor(record=consumer_declaration)) - continue - resource_identity: ResourceIdentity | None = usage.through - if resource_identity is None and isinstance(usage.consumer, ResourceIdentity): - resource_identity = usage.consumer - if resource_identity is not None: - resource: ResourceRecord | None = resources.get(resource_identity) - if resource is not None: - direct.add( - (resource.ownership_root, PurePosixPath(resource.path).parent.as_posix()) - ) - continue - if isinstance(usage.consumer, DeclarationIdentity): - dependents.setdefault(usage.consumer, set()).add(identity) - pending: deque[DeclarationIdentity] = deque(anchors) - while pending: - source: DeclarationIdentity = pending.popleft() - source_anchors: set[_Anchor] = anchors.get(source, set()) - for target in dependents.get(source, ()): - target_anchors: set[_Anchor] = anchors.setdefault(target, set()) - if source_anchors <= target_anchors: - continue - target_anchors |= source_anchors - pending.append(target) - return {identity: frozenset(items) for identity, items in anchors.items()} - - -def _declaration_location_anchor(*, record: DeclarationRecord) -> _Anchor: - """Anchor a use by an audit, schema, or hook at the folder where that declaration lives.""" - - if record.owning_path is None: - return OwnershipRoot(_PROJECT_ROOT_PATH, OwnershipRootKind.GLOBAL), _PROJECT_ROOT_PATH - return record.ownership_root, record.owning_path - - -def _lca(paths: tuple[str, ...]) -> str: - parts: list[tuple[str, ...]] = [PurePosixPath(path).parts for path in paths] - common: list[str] = [] - for components in zip(*parts, strict=False): - if len(set(components)) != 1: - break - common.append(components[0]) - return PurePosixPath(*common).as_posix() - -def _consumer_label(usage: UsageRecord) -> str: - label: str = format_identity(identity=usage.consumer) - if usage.through is not None: - label += f" through {format_identity(identity=usage.through)}" - return label - -def _message( - *, - declaration: DeclarationRecord, - required_scope: ScopeKind, - required_path: str | None, - consumers: str, -) -> str: - current_path: str = declaration.owning_path or declaration.ownership_root.path - directory: str = declaration_directory( - kind=declaration.identity.kind, scope=required_scope, path=required_path - ) - target: str = ( - f"top-level {directory}/" if required_scope is ScopeKind.GLOBAL else f"{directory}/" - ) - consumer_label: str = ( - "References" if declaration.identity.kind is DeclarationKind.SINGULAR_AUDIT else "Consumers" - ) +def _rows(*, index: ScopeIndex) -> NativePlacementRows: return ( - f"Declaration '{format_identity(identity=declaration.identity)}' is currently " - f"{_scope_label(declaration.scope)} at '{current_path}' ({declaration.path}); required " - f"{_scope_label(required_scope)} at '{required_path or 'top-level root'}'. " - f"{consumer_label}: {consumers}. Move it to '{target}'" + [ + ( + (record.identity.kind.value, record.identity.name), + record.path, + root_row(root=record.ownership_root), + ) + for record in index.resources + ], + [ + ( + qualified_row(identity=record.identity), + record.path, + record.line, + record.column, + record.scope.value, + root_row(root=record.ownership_root), + record.owning_path, + ) + for record in index.declarations + ], + [ + ( + qualified_row(identity=usage.consumer), + qualified_row(identity=usage.declaration), + None if usage.through is None else qualified_row(identity=usage.through), + ) + for usage in index.usages + ], ) -def _scope_label(scope: ScopeKind) -> str: - return { - ScopeKind.GLOBAL: "project", - ScopeKind.INHERITED: "descendant-public", - ScopeKind.LOCAL: "exact-owner-private", - ScopeKind.PRIVATE: "model-private", - }[scope] +def qualified_row(*, identity: ResourceIdentity | DeclarationIdentity) -> NativeQualifiedRow: + if isinstance(identity, ResourceIdentity) or identity.owner is None: + return identity.kind.value, identity.name, None + return ( + identity.kind.value, + identity.name, + (identity.owner.kind.value, identity.owner.name), + ) -def _diagnostic( - *, - declaration: DeclarationRecord, - code: ScopeDiagnosticCode, - message: str, - enforce_placement: bool, -) -> ScopeDiagnostic: - return ScopeDiagnostic( - code=code, - message=message, - path=declaration.path, - line=declaration.line, - column=declaration.column, - declaration=declaration.identity, - severity=(DiagnosticSeverity.ERROR if enforce_placement else DiagnosticSeverity.WARNING), +def root_row(*, root: OwnershipRoot) -> NativeRootRow: + return ( + root.path, + root.kind.value, + None if root.resource_kind is None else root.resource_kind.value, ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_operations.py b/src/sqlbuild/compiler/scopes/_helpers/report_operations.py deleted file mode 100644 index 8fe2c646c3..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/report_operations.py +++ /dev/null @@ -1,462 +0,0 @@ -"""Folder browse, move preview, and serialization report operations.""" - -from __future__ import annotations - -import json -from collections.abc import Mapping, Set -from dataclasses import fields, is_dataclass, replace -from enum import Enum -from pathlib import PurePath, PurePosixPath - -from sqlbuild.compiler.scopes._helpers.identities import format_identity -from sqlbuild.compiler.scopes._helpers.paths import is_equal_or_descendant, normalize_path -from sqlbuild.compiler.scopes._helpers.report_projection import ( - declaration_report, - diagnostic_projection, -) -from sqlbuild.compiler.scopes._helpers.report_query import ( - _classify, - _ownership_roots, - _resource_path_is_valid, -) -from sqlbuild.compiler.scopes._helpers.visibility import query_target -from sqlbuild.compiler.scopes.constants import ( - CURRENT_PATH_COMPONENT, - GLOBAL_ALL_POLICY, - KIND_COUNTS_FIELD, - LIST_SECTION, - METADATA_FIELD, - QUALIFIED_IDENTITY_SEPARATOR, -) -from sqlbuild.compiler.scopes.exceptions import ( - InvalidQualifiedIdentityError, - InvalidScopePathError, - ScopeError, -) -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - DeclarationReport, - MovePreview, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeBrowseResult, - ScopeDiagnostic, - ScopeFolder, - ScopeListResult, - ScopeLookup, - ScopeReport, - ScopeReportFilters, - ScopeSection, - ScopeTargetQuery, - VisibilityProvenance, -) -from sqlbuild.compiler.scopes.types import ScopeDiagnosticCode, ScopeKind - - -def browse_folders( - *, - lookup: ScopeLookup, - folder: str = ".", - target: str | PurePath | ResourceIdentity | None = None, - target_is_prospective: bool = False, -) -> ScopeBrowseResult: - """Return only direct child declaration-definition folders.""" - - normalized, diagnostic = _folder_path(folder=folder) - if diagnostic is not None: - return ScopeBrowseResult(folder, (), (diagnostic,), False) - used_identities, target_diagnostics, usage_complete = _target_used_identities( - lookup=lookup, - target=target, - prospective=target_is_prospective, - ) - paths: dict[str, list[DeclarationRecord]] = {} - for record in lookup.index.declarations: - definition_folder: str = _definition_folder(record=record) - parts: tuple[str, ...] = PurePosixPath(definition_folder).parts - parent_parts: tuple[str, ...] = ( - () if normalized == CURRENT_PATH_COMPONENT else PurePosixPath(normalized).parts - ) - if parts[: len(parent_parts)] != parent_parts or len(parts) <= len(parent_parts): - continue - child: str = PurePosixPath(*parts[: len(parent_parts) + 1]).as_posix() - paths.setdefault(child, []).append(record) - folders: tuple[ScopeFolder, ...] = tuple( - _folder_report( - path=path, - records=records, - used_identities=used_identities, - ) - for path, records in sorted(paths.items()) - ) - diagnostics: tuple[ScopeDiagnostic, ...] = ( - *lookup.index.diagnostics, - *target_diagnostics, - ) - return ScopeBrowseResult( - normalized, - folders, - diagnostics, - complete=( - lookup.index.completeness.discovery and usage_complete and not target_diagnostics - ), - ) - - -def list_folder( - *, - lookup: ScopeLookup, - folder: str, - filters: ScopeReportFilters | None = None, - target: str | PurePath | ResourceIdentity | None = None, - target_is_prospective: bool = False, -) -> ScopeListResult: - """List declarations recursively beneath one definition folder.""" - - selected_filters: ScopeReportFilters = filters or ScopeReportFilters(globals=GLOBAL_ALL_POLICY) - normalized, diagnostic = _folder_path(folder=folder) - if diagnostic is not None: - section: ScopeSection = ScopeSection( - LIST_SECTION, 0, 0, complete=False, page_size=selected_filters.page_size - ) - return ScopeListResult(folder, (), section, (diagnostic,)) - used_identities, target_diagnostics, usage_complete = _target_used_identities( - lookup=lookup, - target=target, - prospective=target_is_prospective, - ) - records: list[DeclarationRecord] = [ - record - for record in lookup.index.declarations - if is_equal_or_descendant(path=_definition_folder(record=record), ancestor=normalized) - ] - if selected_filters.kinds: - records = [record for record in records if record.identity.kind in selected_filters.kinds] - if selected_filters.used_only: - records = [record for record in records if record.identity in used_identities] - if selected_filters.defined_under is not None: - try: - under: str = normalize_path(path=selected_filters.defined_under) - records = [ - record - for record in records - if is_equal_or_descendant(path=record.path, ancestor=under) - ] - except InvalidScopePathError: - records = [] - if selected_filters.glob is not None: - from fnmatch import fnmatchcase - - records = [ - record - for record in records - if fnmatchcase(record.identity.name, selected_filters.glob) - or fnmatchcase(format_identity(identity=record.identity), selected_filters.glob) - ] - records.sort(key=lambda item: (format_identity(identity=item.identity), item.path)) - cursor: str | None = selected_filters.cursor - if cursor is not None and QUALIFIED_IDENTITY_SEPARATOR not in cursor: - candidates: list[DeclarationRecord] = [ - item for item in records if item.identity.name == cursor - ] - cursor = format_identity(identity=candidates[0].identity) if len(candidates) == 1 else None - if cursor is None: - diagnostic: ScopeDiagnostic | None = ScopeDiagnostic( - ScopeDiagnosticCode.INVALID_CURSOR, f"Cursor {selected_filters.cursor!r} is invalid" - ) - start: int = 0 - if cursor is not None: - positions: list[int] = [ - index - for index, record in enumerate(records) - if format_identity(identity=record.identity) == cursor - ] - if len(positions) == 1: - start = positions[0] + 1 - else: - diagnostic = ScopeDiagnostic( - ScopeDiagnosticCode.INVALID_CURSOR, f"Cursor {cursor!r} is invalid" - ) - start = 0 - size: int = max(selected_filters.page_size, 0) - selected: list[DeclarationRecord] = records[start : start + size] - reports: tuple[DeclarationReport, ...] = tuple( - declaration_report(lookup=lookup, record=record) for record in selected - ) - truncated: bool = start + len(selected) < len(records) - section: ScopeSection = ScopeSection( - LIST_SECTION, - len(records), - len(reports), - truncated=truncated, - complete=( - lookup.index.completeness.discovery and usage_complete and not target_diagnostics - ), - next_cursor=reports[-1].identity if reports and truncated else None, - cursor=cursor, - page_size=size, - ) - return ScopeListResult( - normalized, - reports, - section, - ( - *lookup.index.diagnostics, - *target_diagnostics, - *((diagnostic,) if diagnostic is not None else ()), - ), - ) - - -def _definition_folder(*, record: DeclarationRecord) -> str: - if record.scope is ScopeKind.GLOBAL: - source_root: str = record.path.split("/", maxsplit=1)[0] - parent: str = PurePosixPath(record.path).parent.as_posix() - relative: str = parent.removeprefix(source_root).strip("/") - base: str = f"global/{record.identity.kind.value}s" - return f"{base}/{relative}" if relative and relative != CURRENT_PATH_COMPONENT else base - return PurePosixPath(record.path).parent.as_posix() - - -def _folder_report( - *, - path: str, - records: list[DeclarationRecord], - used_identities: set[DeclarationIdentity], -) -> ScopeFolder: - child_paths: set[str] = set() - prefix_parts: tuple[str, ...] = PurePosixPath(path).parts - for record in records: - parts: tuple[str, ...] = PurePosixPath(_definition_folder(record=record)).parts - if len(parts) > len(prefix_parts): - child_paths.add(PurePosixPath(*parts[: len(prefix_parts) + 1]).as_posix()) - counts: dict[str, int] = {} - for record in records: - counts[record.identity.kind.value] = counts.get(record.identity.kind.value, 0) + 1 - return ScopeFolder( - path, - PurePosixPath(path).name, - len(records), - len(child_paths), - sum(record.identity in used_identities for record in records), - tuple(sorted(counts.items())), - ) - - -def _target_used_identities( - *, - lookup: ScopeLookup, - target: str | PurePath | ResourceIdentity | None, - prospective: bool, -) -> tuple[set[DeclarationIdentity], tuple[ScopeDiagnostic, ...], bool]: - if prospective: - return set(), (), False - if target is None: - return set(lookup.usages_by_declaration), (), lookup.index.completeness.runtime_usage - try: - query: ScopeTargetQuery = query_target(lookup=lookup, target=target) - except (InvalidQualifiedIdentityError, InvalidScopePathError): - return ( - set(), - ( - ScopeDiagnostic( - ScopeDiagnosticCode.UNKNOWN_TARGET, - f"Browse/list target {target!s} is not a known resource", - ), - ), - False, - ) - if len(query.matches) != 1: - return ( - set(), - ( - ScopeDiagnostic( - ScopeDiagnosticCode.UNKNOWN_TARGET, - f"Browse/list target {target!s} does not resolve to exactly one resource", - ), - ), - False, - ) - return ( - { - usage.declaration - for usage in lookup.usages_by_consumer.get(query.matches[0].identity, ()) - }, - (), - lookup.index.completeness.runtime_usage, - ) - - -def _folder_path(*, folder: str) -> tuple[str, ScopeDiagnostic | None]: - try: - return normalize_path(path=folder), None - except InvalidScopePathError: - return folder, ScopeDiagnostic( - ScopeDiagnosticCode.INVALID_PROSPECTIVE_PATH, - f"Folder {folder!r} is not a project-relative declaration folder", - ) - - -def build_move_preview( - *, lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath -) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: - """Return visibility deltas without mutating the lookup or index.""" - - try: - query: ScopeTargetQuery | None = query_target(lookup=lookup, target=resource) - except (InvalidQualifiedIdentityError, InvalidScopePathError): - query = None - if query is None or len(query.matches) != 1: - return None, ( - ScopeDiagnostic( - ScopeDiagnosticCode.UNKNOWN_TARGET, - f"Move source {resource!s} does not resolve to exactly one resource", - ), - ) - source: ResourceRecord = query.matches[0] - try: - path: str = normalize_path(path=destination) - except InvalidScopePathError: - return None, (_invalid_destination(destination=destination),) - roots: tuple[OwnershipRoot, ...] = _destination_roots(lookup=lookup, source=source, path=path) - occupants: tuple[ResourceRecord, ...] = lookup.resources_by_path.get(path, ()) - occupied: bool = bool(occupants) and path != source.path - if len(roots) != 1 or occupied: - return None, (_invalid_destination(destination=destination),) - moved: ResourceRecord = replace(source, path=path, ownership_root=roots[0]) - old_direct, old_relationships, _old_unavailable = _classify(lookup=lookup, resource=source) - new_direct, new_relationships, _new_unavailable = _classify(lookup=lookup, resource=moved) - old_by_identity: dict[DeclarationIdentity, tuple[DeclarationRecord, VisibilityProvenance]] = { - record.identity: (record, provenance) for record, provenance in old_direct - } - new_by_identity: dict[DeclarationIdentity, tuple[DeclarationRecord, VisibilityProvenance]] = { - record.identity: (record, provenance) for record, provenance in new_direct - } - retained_ids: set[DeclarationIdentity] = set(old_by_identity) & set(new_by_identity) - gained_ids: set[DeclarationIdentity] = set(new_by_identity) - set(old_by_identity) - lost_ids: set[DeclarationIdentity] = set(old_by_identity) - set(new_by_identity) - direct_usages: set[DeclarationIdentity] = { - usage.declaration - for usage in lookup.usages_by_consumer.get(source.identity, ()) - if usage.through is None - } - relationship_ids: set[DeclarationIdentity] = { - record.identity for record, _provenance in old_relationships - } & {record.identity for record, _provenance in new_relationships} - private_ids: set[DeclarationIdentity] = { - record.identity - for record in lookup.index.declarations - if record.scope is ScopeKind.PRIVATE and record.identity.owner == source.identity - } - return ( - MovePreview( - resource=format_identity(identity=source.identity), - destination=path, - new_ownership_root=roots[0].path, - retained=_reports(lookup=lookup, values=old_by_identity, identities=retained_ids), - gained=_reports(lookup=lookup, values=new_by_identity, identities=gained_ids), - lost=_reports(lookup=lookup, values=old_by_identity, identities=lost_ids), - invalidated_usages=tuple( - sorted(format_identity(identity=identity) for identity in lost_ids & direct_usages) - ), - private_retained=_private_reports( - lookup=lookup, - identities=private_ids, - ), - relationship_retained=_reports( - lookup=lookup, - values={item[0].identity: item for item in old_relationships}, - identities=relationship_ids, - ), - complete=( - lookup.index.completeness.static_visibility - and lookup.index.completeness.runtime_usage - and lookup.index.completeness.relationships - ), - ), - (), - ) - - -def serialize_result(*, result: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: - """Serialize a schema-version-two scope result as deterministic ASCII JSON.""" - - payload: object = _value(result) - if not isinstance(payload, dict): - raise ScopeError("Scope report serialization did not produce an object") - payload = {"schema_version": 2, **payload} - return json.dumps(payload, sort_keys=True, ensure_ascii=True, separators=(",", ":")) + "\n" - - -def _reports( - *, - lookup: ScopeLookup, - values: Mapping[DeclarationIdentity, tuple[DeclarationRecord, VisibilityProvenance]], - identities: Set[DeclarationIdentity], -) -> tuple[DeclarationReport, ...]: - reports: list[DeclarationReport] = [] - for identity in sorted(identities, key=lambda item: format_identity(identity=item)): - record, provenance = values[identity] - reports.append(declaration_report(lookup=lookup, record=record, visibility=provenance)) - return tuple(reports) - - -def _destination_roots( - *, lookup: ScopeLookup, source: ResourceRecord, path: str -) -> tuple[OwnershipRoot, ...]: - roots: set[OwnershipRoot] = set() - for root in _ownership_roots(lookup=lookup): - if ( - root.resource_kind == source.identity.kind - and is_equal_or_descendant(path=path, ancestor=root.path) - and _resource_path_is_valid(path=path, root=root) - ): - roots.add(root) - return tuple(sorted(roots, key=lambda item: item.path)) - - -def _private_reports( - *, lookup: ScopeLookup, identities: Set[DeclarationIdentity] -) -> tuple[DeclarationReport, ...]: - reports: list[DeclarationReport] = [] - for identity in sorted(identities, key=lambda item: format_identity(identity=item)): - for record in lookup.declarations.get(identity, ()): - reports.append(declaration_report(lookup=lookup, record=record)) - return tuple(reports) - - -def _invalid_destination(*, destination: str | PurePath) -> ScopeDiagnostic: - return ScopeDiagnostic( - ScopeDiagnosticCode.INVALID_PROSPECTIVE_PATH, - f"Move destination {destination!s} is invalid, outside its authored resource root, " - "or already occupied", - ) - - -def _value(value: object) -> object: - if isinstance(value, ScopeDiagnostic): - return diagnostic_projection(diagnostic=value) - if isinstance(value, Enum): - return value.value - if isinstance(value, PurePath): - return value.as_posix() - if is_dataclass(value) and not isinstance(value, type): - result: dict[str, object] = {} - for item in fields(value): - field_value: object = getattr(value, item.name) - if item.name == METADATA_FIELD and isinstance(field_value, tuple): - result[item.name] = {str(key): _value(member) for key, member in field_value} - elif item.name == KIND_COUNTS_FIELD and isinstance(field_value, tuple): - result[item.name] = {str(key): count for key, count in field_value} - else: - result[item.name] = _value(field_value) - return result - if isinstance(value, tuple | list): - return [_value(item) for item in value] - if isinstance(value, dict): - return {str(key): _value(member) for key, member in value.items()} - if value is None or isinstance(value, bool | int | float | str): - return value - raise ScopeError(f"Unsupported scope report value: {type(value).__name__}") diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_projection.py b/src/sqlbuild/compiler/scopes/_helpers/report_projection.py deleted file mode 100644 index 07d22a6d35..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/report_projection.py +++ /dev/null @@ -1,398 +0,0 @@ -"""Safe deterministic projections for scope query results.""" - -from __future__ import annotations - -from typing import cast - -from sqlbuild.compiler.scopes._helpers.lookup import identity_key -from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes.constants import ( - DECLARATION_GROUP_DIRECTORY, - DECLARATION_ROLE_PARTS, - DEFAULT_ENUM_MEMBER_PREVIEW, - SCOPE_METADATA_SCHEMA_VERSION, -) -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - DeclarationReport, - GrantRecord, - InaccessibleRecord, - ResourceIdentity, - ResourceRecord, - ScopeDiagnostic, - ScopeIndex, - ScopeLookup, - SourceLocation, - UsageRecord, - VisibilityProvenance, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import JsonValue, ScopeKind - - -def build_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: - """Project deterministic value-free metadata from one canonical index.""" - - declarations: list[DeclarationRecord] = sorted( - index.declarations, - key=lambda item: ( - identity_key(item.identity), - item.path, - item.line, - item.column, - ), - ) - resources: list[ResourceRecord] = sorted( - index.resources, - key=lambda item: (identity_key(item.identity), item.path), - ) - grants: list[GrantRecord] = sorted( - index.grants, - key=lambda item: ( - identity_key(item.resource), - identity_key(item.declaration), - identity_key(item.through), - item.kind.value, - ), - ) - usages: list[UsageRecord] = sorted( - index.usages, - key=lambda item: ( - identity_key(item.consumer), - identity_key(item.declaration), - item.kind.value, - identity_key(item.through) if item.through is not None else "", - item.enum_member or "", - ), - ) - visibility: list[VisibilityRecord] = sorted( - index.visibility, - key=lambda item: ( - identity_key(item.resource), - identity_key(item.declaration), - item.reason.value, - identity_key(item.through) if item.through is not None else "", - ), - ) - inaccessible: list[InaccessibleRecord] = sorted( - index.inaccessible, - key=lambda item: ( - identity_key(item.resource), - identity_key(item.declaration), - item.reason.value, - ), - ) - return { - "schema_version": SCOPE_METADATA_SCHEMA_VERSION, - "ownership_roots": [ - { - "path": safe_scope_path(path=root.path), - "resource_kind": ( - root.resource_kind.value if root.resource_kind is not None else None - ), - } - for root in sorted( - index.ownership_roots, - key=lambda item: ( - item.path, - item.resource_kind.value if item.resource_kind is not None else "", - ), - ) - ], - "resources": [ - { - "identity": identity_key(record.identity), - "kind": record.identity.kind.value, - "name": record.identity.name, - "path": safe_scope_path(path=record.path), - "ownership_root": safe_scope_path(path=record.ownership_root.path), - "ownership_root_kind": ( - record.ownership_root.resource_kind.value - if record.ownership_root.resource_kind is not None - else None - ), - } - for record in resources - ], - "declarations": [_declaration_projection(record=record) for record in declarations], - "grants": [ - { - "resource": identity_key(record.resource), - "declaration": identity_key(record.declaration), - "through": identity_key(record.through), - "kind": record.kind.value, - } - for record in grants - ], - "usages": [ - { - "consumer": identity_key(record.consumer), - "declaration": identity_key(record.declaration), - "kind": record.kind.value, - "through": (identity_key(record.through) if record.through is not None else None), - "enum_member": record.enum_member, - } - for record in usages - ], - "visibility": [ - { - "resource": identity_key(record.resource), - "declaration": identity_key(record.declaration), - "reason": record.reason.value, - "through": (identity_key(record.through) if record.through is not None else None), - } - for record in visibility - ], - "inaccessible": [ - { - "resource": identity_key(record.resource), - "declaration": identity_key(record.declaration), - "reason": record.reason.value, - } - for record in inaccessible - ], - "complete": index.completeness.complete, - "completeness": { - section.value: complete for section, complete in index.completeness.as_mapping().items() - }, - "diagnostics": [ - { - "code": diagnostic.code.value, - "message": diagnostic.message.replace("\x1b", ""), - "severity": diagnostic.severity.value, - "path": ( - safe_scope_path(path=diagnostic.path) if diagnostic.path is not None else None - ), - "line": diagnostic.line, - "column": diagnostic.column, - "declaration": ( - identity_key(diagnostic.declaration) - if diagnostic.declaration is not None - else None - ), - "resource": ( - identity_key(diagnostic.resource) if diagnostic.resource is not None else None - ), - } - for diagnostic in sorted( - index.diagnostics, - key=lambda item: ( - item.path or "", - item.line or 0, - item.column or 0, - item.code.value, - ), - ) - ], - } - - -def _declaration_projection(*, record: DeclarationRecord) -> dict[str, JsonValue]: - metadata: dict[str, JsonValue] = {} - if record.macro is not None: - metadata["macro"] = { - "parameters": list(record.macro.parameters), - "dependencies": sorted( - identity_key(identity) for identity in record.macro.dependencies - ), - "source_digest": record.macro.source_digest, - } - if record.enum is not None: - metadata["enum"] = { - "members": [{"name": member.name} for member in record.enum.members], - "scalar_type": record.enum.scalar_type, - } - if record.constant is not None: - metadata["constant"] = { - "logical_type": record.constant.logical_type, - "collection_kind": record.constant.collection_kind, - "item_count": record.constant.item_count, - "nullable": record.constant.nullable, - "render_as": record.constant.render_as, - } - role, visibility, role_root, bucket_path = _declaration_container(record=record) - return { - "identity": identity_key(record.identity), - "kind": record.identity.kind.value, - "name": record.identity.name, - "owner": ( - identity_key(record.identity.owner) if record.identity.owner is not None else None - ), - "path": safe_scope_path(path=record.path), - "line": record.line, - "column": record.column, - "scope": record.scope.value, - "role": role, - "visibility": visibility, - "role_root": role_root, - "bucket_path": bucket_path, - "ownership_root": safe_scope_path(path=record.ownership_root.path), - "owning_path": ( - safe_scope_path(path=record.owning_path) if record.owning_path is not None else None - ), - "metadata": metadata, - } - - -def _declaration_container( - *, record: DeclarationRecord -) -> tuple[str | None, str, str | None, str | None]: - visibility: str = { - ScopeKind.GLOBAL: "project", - ScopeKind.INHERITED: "descendant_public", - ScopeKind.LOCAL: "exact_owner_private", - ScopeKind.PRIVATE: "model_private", - }[record.scope] - if record.scope is ScopeKind.PRIVATE: - return None, visibility, None, None - role_parts: tuple[str, ...] = DECLARATION_ROLE_PARTS[record.identity.kind] - role: str = "/".join(role_parts) - parts: tuple[str, ...] = tuple(safe_scope_path(path=record.path).split("/")) - role_index: int = len(record.owning_path.split("/")) if record.owning_path else 0 - if parts[role_index] == DECLARATION_GROUP_DIRECTORY: - role_index += 1 - role_end: int = role_index + len(role_parts) - role_root: str = "/".join(parts[:role_end]) - bucket: str = "/".join(parts[role_end:-1]) - return role, visibility, role_root, bucket or None - - -def declaration_report( - *, - lookup: ScopeLookup, - record: DeclarationRecord, - visibility: VisibilityProvenance | None = None, - inaccessible_reason: str | None = None, -) -> DeclarationReport: - """Project a declaration without source, values, callables, or process data.""" - - usages: tuple[UsageRecord, ...] = lookup.usages_by_declaration.get(record.identity, ()) - consumers: tuple[str, ...] = tuple(sorted({identity_key(usage.consumer) for usage in usages})) - dependencies: tuple[str, ...] = () - metadata: list[tuple[str, object]] = [] - if record.macro is not None: - dependencies = tuple(sorted(identity_key(item) for item in record.macro.dependencies)) - metadata.extend( - (("parameters", record.macro.parameters), ("dependency_count", len(dependencies))) - ) - if record.enum is not None: - metadata.extend( - ( - ("scalar_type", record.enum.scalar_type), - ( - "members", - tuple( - member.name for member in record.enum.members[:DEFAULT_ENUM_MEMBER_PREVIEW] - ), - ), - ("member_count", len(record.enum.members)), - ( - "members_truncated", - len(record.enum.members) > DEFAULT_ENUM_MEMBER_PREVIEW, - ), - ) - ) - if record.constant is not None: - metadata.extend( - ( - ("logical_type", record.constant.logical_type), - ("collection_kind", record.constant.collection_kind), - ("item_count", record.constant.item_count), - ("nullable", record.constant.nullable), - ("render_as", record.constant.render_as), - ) - ) - role, visibility_label, role_root, bucket_path = _declaration_container(record=record) - metadata.extend( - ( - ("role", role), - ("visibility_label", visibility_label), - ("role_root", role_root), - ("bucket_path", bucket_path), - ) - ) - grants: tuple[str, ...] = tuple( - sorted( - { - identity_key(grant.resource) - for grant in lookup.index.grants - if grant.declaration == record.identity - } - ) - ) - required_scope, required_path, promotion = required_placement(lookup=lookup, record=record) - return DeclarationReport( - identity=identity_key(record.identity), - kind=record.identity.kind.value, - name=record.identity.name, - owner=(identity_key(record.identity.owner) if record.identity.owner is not None else None), - definition=SourceLocation(safe_scope_path(path=record.path), record.line, record.column), - scope=record.scope.value, - owning_path=record.owning_path, - visibility=visibility, - inaccessible_reason=inaccessible_reason, - metadata=tuple(metadata), - consumers=consumers, - dependencies=dependencies, - grants=grants, - required_scope=required_scope, - required_path=required_path, - promotion_impact=promotion, - ) - - -def required_placement( - *, lookup: ScopeLookup, record: DeclarationRecord -) -> tuple[str | None, str | None, tuple[str, ...]]: - """Read exact placement from canonical facts using the placement helper's anchor rules.""" - - if not lookup.usages_by_declaration.get(record.identity): - return None, None, () - from sqlbuild.compiler.scopes._helpers.placement import resolve_required_placement - - resolved: tuple[ScopeKind, str | None, tuple[str, ...]] | None = resolve_required_placement( - index=lookup.index, declaration=record.identity - ) - if resolved is None: - return None, None, () - scope, path, consumers = resolved - current_path: str = record.owning_path or record.ownership_root.path - promotion: tuple[str, ...] = () - if scope.value != record.scope.value or path != current_path: - promotion = consumers - return scope.value, path, promotion - - -def diagnostic_projection(*, diagnostic: ScopeDiagnostic) -> dict[str, object]: - """Return a safe JSON-ready diagnostic projection.""" - - return { - "code": diagnostic.code.value, - "message": diagnostic.message.replace("\x1b", ""), - "severity": diagnostic.severity.value, - "path": safe_scope_path(path=diagnostic.path) if diagnostic.path is not None else None, - "line": diagnostic.line, - "column": diagnostic.column, - "declaration": _identity_text(identity=diagnostic.declaration), - "resource": _identity_text(identity=diagnostic.resource), - } - - -def safe_scope_path(*, path: str) -> str: - """Return a normalized relative path without leaking invalid absolute input.""" - - try: - return normalize_path(path=path) - except InvalidScopePathError: - return "" - - -def _identity_text(*, identity: DeclarationIdentity | object | None) -> str | None: - if identity is None: - return None - typed_identity: DeclarationIdentity | ResourceIdentity = cast( - "DeclarationIdentity | ResourceIdentity", identity - ) - return identity_key(typed_identity) diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_query.py b/src/sqlbuild/compiler/scopes/_helpers/report_query.py deleted file mode 100644 index f34c05cdb4..0000000000 --- a/src/sqlbuild/compiler/scopes/_helpers/report_query.py +++ /dev/null @@ -1,785 +0,0 @@ -"""Pure query engine over the canonical compiler scope lookup.""" - -from __future__ import annotations - -from fnmatch import fnmatchcase -from pathlib import PurePath, PurePosixPath - -from sqlbuild.compiler.scopes._helpers.identities import format_identity, parse_identity -from sqlbuild.compiler.scopes._helpers.paths import is_equal_or_descendant, normalize_path -from sqlbuild.compiler.scopes._helpers.report_projection import declaration_report, safe_scope_path -from sqlbuild.compiler.scopes._helpers.visibility import ( - _inaccessible_reason, - _visibility_reason, - query_target, -) -from sqlbuild.compiler.scopes.constants import ( - AVAILABLE_SECTION, - CURRENT_PATH_COMPONENT, - DECLARATION_KIND_VALUES, - EMPTY_TEXT, - GLOBAL_SUMMARY_POLICY, - GLOBAL_USED_POLICY, - PATH_SEPARATOR, - QUALIFIED_IDENTITY_SEPARATOR, - RELATIONSHIP_SECTION, - USED_SECTION, - WINDOWS_PATH_SEPARATOR, -) -from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError, InvalidScopePathError -from sqlbuild.compiler.scopes.models import ( - DeclarationExplanation, - DeclarationIdentity, - DeclarationRecord, - DeclarationReport, - GrantRecord, - OwnershipRoot, - ResourceIdentity, - ResourceRecord, - ScopeChainEntry, - ScopeDiagnostic, - ScopeLookup, - ScopeReport, - ScopeReportFilters, - ScopeResourceReport, - ScopeSection, - ScopeTargetQuery, - UsageRecord, - VisibilityProvenance, -) -from sqlbuild.compiler.scopes.types import ( - GrantKind, - ResourceKind, - ScopeDiagnosticCode, - ScopeKind, - VisibilityReason, -) - - -def build_scope_report( - *, - lookup: ScopeLookup, - target: str | PurePath | ResourceIdentity | DeclarationIdentity | None = None, - at: str | PurePath | None = None, - directory: bool = False, - filters: ScopeReportFilters | None = None, -) -> ScopeReport: - """Build a complete value-safe report for an existing target or prospective path.""" - - selected_filters: ScopeReportFilters = filters or ScopeReportFilters() - diagnostics: list[ScopeDiagnostic] = list(lookup.index.diagnostics) - if at is not None: - resource, record, target_diagnostics = _prospective_resource( - lookup=lookup, at=at, directory=directory - ) - diagnostics.extend(target_diagnostics) - explanation: DeclarationExplanation | None = None - else: - resource, record, explanation, target_diagnostics = _existing_target( - lookup=lookup, target=target - ) - diagnostics.extend(target_diagnostics) - if record is None: - explanation_complete: bool = explanation is not None and explanation.complete - return ScopeReport( - resource=resource, - filters=selected_filters, - explanation=explanation, - diagnostics=tuple(diagnostics), - complete=explanation_complete, - ) - - direct, relationships, unavailable = _classify(lookup=lookup, resource=record) - target_usages: tuple[UsageRecord, ...] = lookup.usages_by_consumer.get(record.identity, ()) - used_identities: set[DeclarationIdentity] = {usage.declaration for usage in target_usages} - direct_used: set[DeclarationIdentity] = { - usage.declaration for usage in target_usages if usage.through is None - } - relationship_used: set[DeclarationIdentity] = { - usage.declaration for usage in target_usages if usage.through is not None - } - used_by_identity: dict[DeclarationIdentity, tuple[DeclarationRecord, VisibilityProvenance]] = { - item[0].identity: item for item in direct if item[0].identity in direct_used - } - used_by_identity.update( - {item[0].identity: item for item in relationships if item[0].identity in relationship_used} - ) - used_records: tuple[tuple[DeclarationRecord, VisibilityProvenance], ...] = tuple( - sorted(used_by_identity.values(), key=_declaration_pair_key) - ) - nearby: tuple[tuple[DeclarationRecord, str], ...] = _nearby( - lookup=lookup, - resource=record, - unavailable=unavailable, - depth=selected_filters.nearby_depth, - ) - - direct: tuple[tuple[DeclarationRecord, VisibilityProvenance], ...] = _filter_visible( - lookup=lookup, - values=direct, - filters=selected_filters, - used_identities=used_identities, - ) - used_records = _filter_visible( - lookup=lookup, - values=used_records, - filters=selected_filters, - used_identities=used_identities, - ) - relationships: tuple[tuple[DeclarationRecord, VisibilityProvenance], ...] = _filter_visible( - lookup=lookup, - values=relationships, - filters=selected_filters, - used_identities=used_identities, - ) - nearby = _filter_nearby(values=nearby, filters=selected_filters, used=used_identities) - - if AVAILABLE_SECTION not in selected_filters.sections: - direct = () - if USED_SECTION not in selected_filters.sections: - used_records = () - if RELATIONSHIP_SECTION not in selected_filters.sections: - relationships = () - - if selected_filters.dependency_depth > 0: - used_records = _expand_dependencies( - lookup=lookup, values=used_records, depth=selected_filters.dependency_depth - ) - - collapsed: int = 0 - if selected_filters.globals == GLOBAL_SUMMARY_POLICY: - retained: tuple[tuple[DeclarationRecord, VisibilityProvenance], ...] = tuple( - item - for item in direct - if item[0].scope is not ScopeKind.GLOBAL or item[0].identity in used_identities - ) - collapsed = len(direct) - len(retained) - direct = retained - elif selected_filters.globals == GLOBAL_USED_POLICY: - direct = tuple( - item - for item in direct - if item[0].scope is not ScopeKind.GLOBAL or item[0].identity in used_identities - ) - - report_sections: tuple[tuple[tuple[DeclarationRecord, VisibilityProvenance], ...], ...] = ( - direct, - used_records, - relationships, - ) - cursor, cursor_diagnostic = _resolve_cursor( - requested=selected_filters.cursor, - sections=report_sections, - ) - if cursor_diagnostic is not None: - diagnostics.append(cursor_diagnostic) - available, available_section = _page( - lookup=lookup, - name=AVAILABLE_SECTION, - values=direct, - cursor=cursor, - page_size=selected_filters.page_size, - complete=lookup.index.completeness.static_visibility, - collapsed=collapsed, - ) - used, used_section = _page( - lookup=lookup, - name=USED_SECTION, - values=used_records, - cursor=cursor, - page_size=selected_filters.page_size, - complete=lookup.index.completeness.runtime_usage, - ) - relationship_scope, relationship_section = _page( - lookup=lookup, - name=RELATIONSHIP_SECTION, - values=relationships, - cursor=cursor, - page_size=selected_filters.page_size, - complete=lookup.index.completeness.relationships, - ) - nearby_reports: tuple[DeclarationReport, ...] = tuple( - declaration_report( - lookup=lookup, - record=item, - inaccessible_reason=reason, - ) - for item, reason in nearby[: selected_filters.page_size] - ) - nearby_section: ScopeSection = ScopeSection( - name="nearby_unavailable", - total=len(nearby), - returned=len(nearby_reports), - truncated=len(nearby_reports) < len(nearby), - complete=lookup.index.completeness.static_visibility, - next_cursor=(nearby_reports[-1].identity if len(nearby_reports) < len(nearby) else None), - page_size=selected_filters.page_size, - ) - sections: tuple[ScopeSection, ...] = ( - available_section, - used_section, - relationship_section, - nearby_section, - ) - prospective_complete: bool = not resource.prospective - complete: bool = all(section.complete for section in sections) and prospective_complete - if resource.prospective: - diagnostics.append( - ScopeDiagnostic( - code=ScopeDiagnosticCode.INCOMPLETE_USAGE, - message=( - "Runtime usage and relationship facts are unavailable for a prospective path" - ), - path=resource.path, - ) - ) - return ScopeReport( - resource=resource, - scope_chain=_scope_chain(lookup=lookup, resource=record), - available=available, - used=used, - relationship_scope=relationship_scope, - nearby_unavailable=nearby_reports if selected_filters.include_nearby else (), - filters=selected_filters, - sections=sections, - explanation=explanation, - diagnostics=tuple(diagnostics), - complete=complete, - ) - - -def explain_declaration( - *, lookup: ScopeLookup, target: str | DeclarationIdentity, at: ResourceRecord | None = None -) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: - """Explain one required kind-qualified declaration identity.""" - - diagnostics: list[ScopeDiagnostic] = [] - identity: DeclarationIdentity | None = ( - target if isinstance(target, DeclarationIdentity) else None - ) - if identity is None: - try: - parsed: ResourceIdentity | DeclarationIdentity = parse_identity(value=str(target)) - identity = parsed if isinstance(parsed, DeclarationIdentity) else None - except InvalidQualifiedIdentityError: - identity = None - if identity is None: - diagnostics.append( - _diagnostic(code=ScopeDiagnosticCode.UNQUALIFIED_TARGET, value=str(target)) - ) - return DeclarationExplanation(None, False), tuple(diagnostics) - records: tuple[DeclarationRecord, ...] = lookup.declarations.get(identity, ()) - if not records: - diagnostics.append( - _diagnostic( - code=ScopeDiagnosticCode.UNKNOWN_TARGET, - value=format_identity(identity=identity), - ) - ) - return DeclarationExplanation(None, False), tuple(diagnostics) - if len(records) > 1: - diagnostics.append( - _diagnostic( - code=ScopeDiagnosticCode.DUPLICATE_DECLARATION, - value=format_identity(identity=identity), - ) - ) - record: DeclarationRecord = records[0] - visibility: VisibilityProvenance | None = None - inaccessible: str | None = None - if at is not None: - reason: VisibilityReason | None = _visibility_reason(resource=at, declaration=record) - if reason is None: - matching_grants: tuple[GrantRecord, ...] = tuple( - grant - for grant in lookup.grants_by_resource.get(at.identity, ()) - if grant.declaration == record.identity - ) - if matching_grants: - grant_reason: VisibilityReason = ( - VisibilityReason.TESTED_MACRO - if all(grant.kind is GrantKind.TESTED_MACRO for grant in matching_grants) - else VisibilityReason.EXPECTED_MODEL - ) - visibility = VisibilityProvenance( - grant_reason.value, - ",".join( - sorted(format_identity(identity=grant.through) for grant in matching_grants) - ), - ) - else: - inaccessible = _inaccessible_reason(resource=at, declaration=record).value - else: - visibility = VisibilityProvenance(reason.value) - return ( - DeclarationExplanation( - declaration_report( - lookup=lookup, - record=record, - visibility=visibility, - inaccessible_reason=inaccessible, - ), - lookup.index.completeness.placement, - ), - tuple(diagnostics), - ) - - -def _existing_target( - *, - lookup: ScopeLookup, - target: str | PurePath | ResourceIdentity | DeclarationIdentity | None, -) -> tuple[ - ScopeResourceReport, - ResourceRecord | None, - DeclarationExplanation | None, - tuple[ScopeDiagnostic, ...], -]: - if target is None: - diagnostic: ScopeDiagnostic = _diagnostic( - code=ScopeDiagnosticCode.UNQUALIFIED_TARGET, value=EMPTY_TEXT - ) - return ScopeResourceReport(EMPTY_TEXT, None, None), None, None, (diagnostic,) - raw: str = ( - format_identity(identity=target) if not isinstance(target, (str, PurePath)) else str(target) - ) - if isinstance(target, DeclarationIdentity) or ( - isinstance(target, str) - and target.split(QUALIFIED_IDENTITY_SEPARATOR, maxsplit=1)[0] in DECLARATION_KIND_VALUES - ): - explanation, explain_diagnostics = explain_declaration(lookup=lookup, target=target) - return ScopeResourceReport(raw, raw, None), None, explanation, explain_diagnostics - if ( - isinstance(target, str) - and QUALIFIED_IDENTITY_SEPARATOR not in target - and not _looks_like_path(value=target) - ): - diagnostic = _diagnostic(code=ScopeDiagnosticCode.UNQUALIFIED_TARGET, value=raw) - return ScopeResourceReport(raw, None, None), None, None, (diagnostic,) - try: - query: ScopeTargetQuery = query_target(lookup=lookup, target=target) - except (InvalidQualifiedIdentityError, InvalidScopePathError): - diagnostic = _unknown_target_diagnostic(value=raw) - return ScopeResourceReport(raw, None, None), None, None, (diagnostic,) - if query.unknown: - diagnostic = _unknown_target_diagnostic(value=raw) - return ScopeResourceReport(raw, None, None), None, None, (diagnostic,) - target_diagnostics: tuple[ScopeDiagnostic, ...] = () - if len(query.matches) > 1: - target_diagnostics = ( - ScopeDiagnostic( - ScopeDiagnosticCode.DUPLICATE_RESOURCE, - f"Target {raw!r} matches {len(query.matches)} resources", - ), - ) - record: ResourceRecord = query.matches[0] - return ( - ScopeResourceReport( - raw, - format_identity(identity=record.identity), - safe_scope_path(path=record.path), - duplicate_count=len(query.matches), - ), - record, - None, - target_diagnostics, - ) - - -def _prospective_resource( - *, - lookup: ScopeLookup, - at: str | PurePath, - directory: bool, -) -> tuple[ScopeResourceReport, ResourceRecord | None, tuple[ScopeDiagnostic, ...]]: - raw: str = str(at) - try: - path: str = normalize_path(path=at) - except InvalidScopePathError: - diagnostic: ScopeDiagnostic = _diagnostic( - code=ScopeDiagnosticCode.INVALID_PROSPECTIVE_PATH, value=raw - ) - return ScopeResourceReport(raw, None, None, True, directory), None, (diagnostic,) - roots: tuple[OwnershipRoot, ...] = _ownership_roots(lookup=lookup) - root: OwnershipRoot | None = next( - (item for item in roots if is_equal_or_descendant(path=path, ancestor=item.path)), - None, - ) - if root is None or (not directory and not _resource_path_is_valid(path=path, root=root)): - diagnostic = _diagnostic(code=ScopeDiagnosticCode.INVALID_PROSPECTIVE_PATH, value=raw) - return ScopeResourceReport(raw, None, path, True, directory), None, (diagnostic,) - authored_path: str = f"{path}/{_default_suffix(root=root)}" if directory else path - identity: ResourceIdentity = ResourceIdentity( - root.resource_kind or ResourceKind.MODEL, f"" - ) - record: ResourceRecord = ResourceRecord(identity, authored_path, root) - return ScopeResourceReport(raw, None, path, True, directory), record, () - - -def _classify( - *, lookup: ScopeLookup, resource: ResourceRecord -) -> tuple[ - tuple[tuple[DeclarationRecord, VisibilityProvenance], ...], - tuple[tuple[DeclarationRecord, VisibilityProvenance], ...], - tuple[tuple[DeclarationRecord, str], ...], -]: - grants: tuple[GrantRecord, ...] = lookup.grants_by_resource.get(resource.identity, ()) - granted: dict[DeclarationIdentity, tuple[GrantRecord, ...]] = {} - for grant in grants: - granted.setdefault(grant.declaration, ()) - granted[grant.declaration] += (grant,) - direct: list[tuple[DeclarationRecord, VisibilityProvenance]] = [] - relationships: list[tuple[DeclarationRecord, VisibilityProvenance]] = [] - unavailable: list[tuple[DeclarationRecord, str]] = [] - for declaration in lookup.index.declarations: - reason: VisibilityReason | None = _visibility_reason( - resource=resource, declaration=declaration - ) - if reason is not None: - direct.append((declaration, VisibilityProvenance(reason.value))) - if declaration.identity in granted and declaration.scope is not ScopeKind.PRIVATE: - declaration_grants: tuple[GrantRecord, ...] = granted[declaration.identity] - grant_reason: VisibilityReason = ( - VisibilityReason.TESTED_MACRO - if all(grant.kind is GrantKind.TESTED_MACRO for grant in declaration_grants) - else VisibilityReason.EXPECTED_MODEL - ) - relationships.append( - ( - declaration, - VisibilityProvenance( - grant_reason.value, - ",".join( - sorted( - format_identity(identity=grant.through) - for grant in declaration_grants - ) - ), - ), - ) - ) - elif reason is None: - unavailable.append( - ( - declaration, - _inaccessible_reason(resource=resource, declaration=declaration).value, - ) - ) - return ( - tuple(sorted(direct, key=_declaration_pair_key)), - tuple(sorted(relationships, key=_declaration_pair_key)), - tuple(sorted(unavailable, key=_declaration_pair_key)), - ) - - -def _filter_visible( - *, - lookup: ScopeLookup, - values: tuple[tuple[DeclarationRecord, VisibilityProvenance], ...], - filters: ScopeReportFilters, - used_identities: set[DeclarationIdentity], -) -> tuple[tuple[DeclarationRecord, VisibilityProvenance], ...]: - selected: tuple[tuple[DeclarationRecord, VisibilityProvenance], ...] = values - if filters.defined_under is not None: - try: - under: str = normalize_path(path=filters.defined_under) - selected = tuple( - item - for item in selected - if is_equal_or_descendant(path=item[0].path, ancestor=under) - ) - except InvalidScopePathError: - selected = () - if filters.kinds: - selected = tuple(item for item in selected if item[0].identity.kind in filters.kinds) - if filters.glob is not None: - selected = tuple( - item - for item in selected - if fnmatchcase(item[0].identity.name, filters.glob) - or fnmatchcase(format_identity(identity=item[0].identity), filters.glob) - ) - if filters.used_only: - selected = tuple(item for item in selected if item[0].identity in used_identities) - return selected - - -def _filter_nearby( - *, - values: tuple[tuple[DeclarationRecord, str], ...], - filters: ScopeReportFilters, - used: set[DeclarationIdentity], -) -> tuple[tuple[DeclarationRecord, str], ...]: - selected: tuple[tuple[DeclarationRecord, str], ...] = values - if filters.defined_under is not None: - try: - under: str = normalize_path(path=filters.defined_under) - selected = tuple( - item - for item in selected - if is_equal_or_descendant(path=item[0].path, ancestor=under) - ) - except InvalidScopePathError: - selected = () - if filters.kinds: - selected = tuple(item for item in selected if item[0].identity.kind in filters.kinds) - if filters.glob is not None: - selected = tuple( - item - for item in selected - if fnmatchcase(item[0].identity.name, filters.glob) - or fnmatchcase(format_identity(identity=item[0].identity), filters.glob) - ) - if filters.used_only: - selected = tuple(item for item in selected if item[0].identity in used) - return selected - - -def _expand_dependencies( - *, - lookup: ScopeLookup, - values: tuple[tuple[DeclarationRecord, VisibilityProvenance], ...], - depth: int, -) -> tuple[tuple[DeclarationRecord, VisibilityProvenance], ...]: - selected: dict[DeclarationIdentity, tuple[DeclarationRecord, VisibilityProvenance]] = { - record.identity: (record, provenance) for record, provenance in values - } - frontier: set[DeclarationIdentity] = set(selected) - for _level in range(max(depth, 0)): - dependencies: set[DeclarationIdentity] = set() - for identity in frontier: - for record in lookup.declarations.get(identity, ()): - if record.macro is not None: - dependencies.update(record.macro.dependencies) - dependencies.update( - usage.declaration for usage in lookup.usages_by_consumer.get(identity, ()) - ) - frontier = dependencies - set(selected) - for identity in frontier: - for record in lookup.declarations.get(identity, ()): - selected[identity] = (record, VisibilityProvenance("dependency")) - return tuple( - sorted( - selected.values(), - key=lambda item: (format_identity(identity=item[0].identity), item[0].path), - ) - ) - - -def _nearby( - *, - lookup: ScopeLookup, - resource: ResourceRecord, - unavailable: tuple[tuple[DeclarationRecord, str], ...], - depth: int, -) -> tuple[tuple[DeclarationRecord, str], ...]: - parent: str = PurePosixPath(resource.path).parent.as_posix() - parent_parts: tuple[str, ...] = PurePosixPath(parent).parts - relationship_declarations: set[DeclarationIdentity] = { - grant.declaration for grant in lookup.grants_by_resource.get(resource.identity, ()) - } - result: list[tuple[DeclarationRecord, str]] = [] - for record, reason in unavailable: - if record.scope is ScopeKind.PRIVATE and record.identity.owner != resource.identity: - continue - owner: str = record.owning_path or record.ownership_root.path - owner_parts: tuple[str, ...] = PurePosixPath(owner).parts - same_root: bool = record.ownership_root.path == resource.ownership_root.path - ancestor: bool = is_equal_or_descendant(path=parent, ancestor=owner) - connected: bool = record.identity in relationship_declarations - sibling: bool = ( - max(depth, 0) >= 1 and PurePosixPath(owner).parent == PurePosixPath(parent).parent - ) - descendant: bool = is_equal_or_descendant(path=owner, ancestor=parent) and ( - len(owner_parts) - len(parent_parts) <= max(depth, 0) - ) - immediate_relative: bool = sibling or descendant - if connected or (same_root and (ancestor or immediate_relative)): - result.append((record, reason)) - return tuple(result) - - -def _scope_chain(*, lookup: ScopeLookup, resource: ResourceRecord) -> tuple[ScopeChainEntry, ...]: - parent: str = PurePosixPath(resource.path).parent.as_posix() - paths: set[str] = {parent} - current: PurePosixPath = PurePosixPath(parent) - while current.as_posix() not in {CURRENT_PATH_COMPONENT, resource.ownership_root.path}: - current = current.parent - paths.add(current.as_posix()) - paths.add(resource.ownership_root.path) - entries: list[ScopeChainEntry] = [] - for path in sorted(paths, key=lambda item: (-len(PurePosixPath(item).parts), item)): - count: int = sum( - 1 - for declaration in lookup.index.declarations - if declaration.scope is not ScopeKind.GLOBAL - and (declaration.owning_path or declaration.ownership_root.path) == path - ) - entries.append(ScopeChainEntry("local" if path == parent else "inherited", path, count)) - global_count: int = sum( - 1 for declaration in lookup.index.declarations if declaration.scope is ScopeKind.GLOBAL - ) - entries.append(ScopeChainEntry("global", "global", global_count)) - return tuple(entries) - - -def _resolve_cursor( - *, - requested: str | None, - sections: tuple[tuple[tuple[DeclarationRecord, VisibilityProvenance], ...], ...], -) -> tuple[str | None, ScopeDiagnostic | None]: - if requested is None: - return None, None - identities: tuple[str, ...] = _cursor_identities(sections=sections) - unique_identities: set[str] = set(identities) - section_match_counts: tuple[int, ...] = _section_identity_counts( - sections=sections, identity=requested - ) - if ( - QUALIFIED_IDENTITY_SEPARATOR in requested - and requested in unique_identities - and max(section_match_counts, default=0) == 1 - ): - return requested, None - bare_matches: tuple[str, ...] = tuple( - identity - for identity in sorted(unique_identities) - if identity.rsplit(QUALIFIED_IDENTITY_SEPARATOR, 1)[-1] == requested - ) - if QUALIFIED_IDENTITY_SEPARATOR not in requested and len(bare_matches) == 1: - resolved: str = next( - identity - for identity in unique_identities - if identity.rsplit(QUALIFIED_IDENTITY_SEPARATOR, 1)[-1] == requested - ) - resolved_counts: tuple[int, ...] = _section_identity_counts( - sections=sections, identity=resolved - ) - if max(resolved_counts, default=0) == 1: - return resolved, None - diagnostic: ScopeDiagnostic = _diagnostic( - code=ScopeDiagnosticCode.INVALID_CURSOR, value=requested - ) - return None, diagnostic - - -def _page( - *, - lookup: ScopeLookup, - name: str, - values: tuple[tuple[DeclarationRecord, VisibilityProvenance], ...], - cursor: str | None, - page_size: int, - complete: bool, - collapsed: int = 0, -) -> tuple[tuple[DeclarationReport, ...], ScopeSection]: - size: int = max(page_size, 0) - start: int = 0 - if cursor is not None: - start = next( - ( - index + 1 - for index, item in enumerate(values) - if format_identity(identity=item[0].identity) == cursor - ), - 0, - ) - selected: tuple[tuple[DeclarationRecord, VisibilityProvenance], ...] = values[ - start : start + size - ] - reports: tuple[DeclarationReport, ...] = tuple( - declaration_report(lookup=lookup, record=record, visibility=provenance) - for record, provenance in selected - ) - truncated: bool = start + len(selected) < len(values) - return reports, ScopeSection( - name=name, - total=len(values) + collapsed, - returned=len(reports), - collapsed=collapsed > 0, - collapsed_count=collapsed, - truncated=truncated, - complete=complete, - next_cursor=reports[-1].identity if reports and truncated else None, - cursor=cursor, - page_size=size, - ) - - -def _looks_like_path(*, value: str) -> bool: - return ( - PATH_SEPARATOR in value - or WINDOWS_PATH_SEPARATOR in value - or PurePosixPath(value).suffix != EMPTY_TEXT - ) - - -def _ownership_roots(*, lookup: ScopeLookup) -> tuple[OwnershipRoot, ...]: - roots: set[OwnershipRoot] = set(lookup.index.ownership_roots) - roots.update(record.ownership_root for record in lookup.index.resources) - return tuple( - sorted( - roots, - key=lambda item: ( - -len(PurePosixPath(item.path).parts), - item.path, - item.resource_kind.value if item.resource_kind is not None else "", - ), - ) - ) - - -def _resource_path_is_valid(*, path: str, root: OwnershipRoot) -> bool: - if path == root.path: - return False - return PurePosixPath(path).suffix.lower() in _valid_suffixes(root=root) - - -def _default_suffix(*, root: OwnershipRoot) -> str: - return ".yml" if root.resource_kind is ResourceKind.SOURCE else ".sql" - - -def _valid_suffixes(*, root: OwnershipRoot) -> frozenset[str]: - if root.resource_kind is ResourceKind.SOURCE: - return frozenset({".yml", ".yaml"}) - return frozenset({_default_suffix(root=root)}) - - -def _declaration_pair_key[Provenance]( - item: tuple[DeclarationRecord, Provenance], -) -> tuple[str, str]: - return format_identity(identity=item[0].identity), item[0].path - - -def _cursor_identities( - *, sections: tuple[tuple[tuple[DeclarationRecord, VisibilityProvenance], ...], ...] -) -> tuple[str, ...]: - identities: list[str] = [] - for section in sections: - identities.extend(format_identity(identity=item[0].identity) for item in section) - return tuple(identities) - - -def _section_identity_counts( - *, - sections: tuple[tuple[tuple[DeclarationRecord, VisibilityProvenance], ...], ...], - identity: str, -) -> tuple[int, ...]: - counts: list[int] = [] - for section in sections: - count: int = sum(format_identity(identity=item[0].identity) == identity for item in section) - counts.append(count) - return tuple(counts) - - -def _diagnostic(*, code: ScopeDiagnosticCode, value: str) -> ScopeDiagnostic: - label: str = "Target" if code is not ScopeDiagnosticCode.INVALID_CURSOR else "Cursor" - return ScopeDiagnostic(code, f"{label} {value!r} is not valid for this scope query") - - -def _unknown_target_diagnostic(*, value: str) -> ScopeDiagnostic: - target_type: str = "path" if _looks_like_path(value=value) else "qualified identity" - return ScopeDiagnostic( - ScopeDiagnosticCode.UNKNOWN_TARGET, - f"Unknown {target_type} target {value!r}", - ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/report_rows.py b/src/sqlbuild/compiler/scopes/_helpers/report_rows.py new file mode 100644 index 0000000000..e4e9aa297b --- /dev/null +++ b/src/sqlbuild/compiler/scopes/_helpers/report_rows.py @@ -0,0 +1,274 @@ +"""Report models and scope indexes as the rows native serialization and projection read.""" + +from __future__ import annotations + +from enum import Enum + +import sqlbuild._native as _native +from sqlbuild.compiler.scopes._helpers.placement import qualified_row, root_row +from sqlbuild.compiler.scopes._helpers.reports import filters_row, values_row +from sqlbuild.compiler.scopes.exceptions import ScopeError +from sqlbuild.compiler.scopes.models import ( + DeclarationExplanation, + DeclarationReport, + MovePreview, + ScopeBrowseResult, + ScopeCompleteness, + ScopeDiagnostic, + ScopeFolder, + ScopeIndex, + ScopeListResult, + ScopeReport, + ScopeResourceReport, + ScopeSection, +) +from sqlbuild.compiler.scopes.types import ( + JsonValue, + NativeScopeDeclarationRow, + NativeScopeDiagnosticRow, + NativeScopeExplanationRow, + NativeScopeFolderRow, + NativeScopeMoveRow, + NativeScopeSectionRow, +) + + +def serialize_result(*, result: ScopeReport | ScopeBrowseResult | ScopeListResult) -> str: + """Deterministic schema-version-two ASCII JSON of one scope query result.""" + + try: + if isinstance(result, ScopeReport): + return _native.serialize_scope_result_row("report", _report_row(report=result)) + if isinstance(result, ScopeBrowseResult): + return _native.serialize_scope_result_row( + "browse", + ( + result.folder, + [_folder_row(folder=folder) for folder in result.folders], + _diagnostic_rows(diagnostics=result.diagnostics), + result.complete, + ), + ) + return _native.serialize_scope_result_row( + "list", + ( + result.folder, + _declaration_rows(reports=result.declarations), + _section_row(section=result.section), + _diagnostic_rows(diagnostics=result.diagnostics), + ), + ) + except ValueError as error: + raise ScopeError(str(error)) from error + + +def build_projection(*, index: ScopeIndex) -> dict[str, JsonValue]: + """Deterministic value-free metadata of one canonical index.""" + + completeness: ScopeCompleteness = index.completeness + return _native.scope_metadata_projection_rows( + ( + [root_row(root=root) for root in index.ownership_roots], + [ + ( + (record.identity.kind.value, record.identity.name), + record.path, + root_row(root=record.ownership_root), + ) + for record in index.resources + ], + [ + ( + qualified_row(identity=record.identity), + record.path, + record.line, + record.column, + record.scope.value, + root_row(root=record.ownership_root), + record.owning_path, + ) + for record in index.declarations + ], + [values_row(record=record) for record in index.declarations], + [ + ( + qualified_row(identity=usage.consumer), + qualified_row(identity=usage.declaration), + usage.kind.value, + None if usage.through is None else qualified_row(identity=usage.through), + usage.enum_member, + ) + for usage in index.usages + ], + [ + ( + (grant.resource.kind.value, grant.resource.name), + qualified_row(identity=grant.declaration), + qualified_row(identity=grant.through), + grant.kind.value, + ) + for grant in index.grants + ], + [ + ( + (fact.resource.kind.value, fact.resource.name), + qualified_row(identity=fact.declaration), + fact.reason.value, + None if fact.through is None else qualified_row(identity=fact.through), + ) + for fact in index.visibility + ], + [ + ( + (fact.resource.kind.value, fact.resource.name), + qualified_row(identity=fact.declaration), + fact.reason.value, + None, + ) + for fact in index.inaccessible + ], + _diagnostic_rows(diagnostics=index.diagnostics), + ( + completeness.discovery, + completeness.static_visibility, + completeness.runtime_usage, + completeness.relationships, + completeness.placement, + completeness.promotion_impact, + ), + ) + ) + + +def _folder_row(*, folder: ScopeFolder) -> NativeScopeFolderRow: + return ( + folder.path, + folder.name, + folder.descendant_count, + folder.child_count, + folder.used_count, + [(str(kind), count) for kind, count in folder.kind_counts], + ) + + +def _text(value: object) -> str: + return str(value.value) if isinstance(value, Enum) else str(value) + + +def _diagnostic_rows(*, diagnostics: tuple[ScopeDiagnostic, ...]) -> list[NativeScopeDiagnosticRow]: + return [ + ( + _text(diagnostic.code), + diagnostic.message, + _text(diagnostic.severity), + diagnostic.path, + diagnostic.line, + diagnostic.column, + None + if diagnostic.declaration is None + else qualified_row(identity=diagnostic.declaration), + ( + None + if diagnostic.resource is None + else (diagnostic.resource.kind.value, diagnostic.resource.name) + ), + ) + for diagnostic in diagnostics + ] + + +def _declaration_row(report: DeclarationReport) -> NativeScopeDeclarationRow: + return ( + ( + report.identity, + report.kind, + report.name, + report.owner, + (report.definition.path, report.definition.line, report.definition.column), + report.scope, + report.owning_path, + ), + ( + ( + None + if report.visibility is None + else (report.visibility.reason, report.visibility.through) + ), + report.inaccessible_reason, + [(str(key), value) for key, value in report.metadata], + ), + ( + list(report.consumers), + list(report.dependencies), + list(report.grants), + report.required_scope, + report.required_path, + list(report.promotion_impact), + ), + ) + + +def _declaration_rows(*, reports: tuple[DeclarationReport, ...]) -> list[NativeScopeDeclarationRow]: + return [_declaration_row(report) for report in reports] + + +def _section_row(*, section: ScopeSection) -> NativeScopeSectionRow: + return ( + section.name, + section.total, + section.returned, + section.collapsed, + section.collapsed_count, + section.truncated, + section.complete, + section.next_cursor, + section.cursor, + section.page_size, + ) + + +def _explanation_row(*, explanation: DeclarationExplanation) -> NativeScopeExplanationRow: + return ( + None if explanation.declaration is None else _declaration_row(explanation.declaration), + explanation.complete, + ) + + +def _move_row(*, move: MovePreview) -> NativeScopeMoveRow: + return ( + move.resource, + move.destination, + move.new_ownership_root, + _declaration_rows(reports=move.retained), + _declaration_rows(reports=move.gained), + _declaration_rows(reports=move.lost), + list(move.invalidated_usages), + _declaration_rows(reports=move.private_retained), + _declaration_rows(reports=move.relationship_retained), + move.complete, + ) + + +def _report_row(*, report: ScopeReport) -> object: + resource: ScopeResourceReport = report.resource + return ( + ( + resource.target, + resource.identity, + resource.path, + resource.prospective, + resource.directory, + resource.duplicate_count, + ), + [(entry.kind, entry.path, entry.declaration_count) for entry in report.scope_chain], + _declaration_rows(reports=report.available), + _declaration_rows(reports=report.used), + _declaration_rows(reports=report.relationship_scope), + _declaration_rows(reports=report.nearby_unavailable), + filters_row(filters=report.filters), + [_section_row(section=section) for section in report.sections], + None if report.explanation is None else _explanation_row(explanation=report.explanation), + None if report.move_preview is None else _move_row(move=report.move_preview), + _diagnostic_rows(diagnostics=report.diagnostics), + report.complete, + ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/reports.py b/src/sqlbuild/compiler/scopes/_helpers/reports.py new file mode 100644 index 0000000000..a1ffb3a3ce --- /dev/null +++ b/src/sqlbuild/compiler/scopes/_helpers/reports.py @@ -0,0 +1,371 @@ +"""Scope reports answered by the native scope report engine, projected as report models.""" + +from __future__ import annotations + +from pathlib import PurePath +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.scopes._helpers.placement import qualified_row +from sqlbuild.compiler.scopes.exceptions import ( + InvalidQualifiedIdentityError, + InvalidScopePathError, +) +from sqlbuild.compiler.scopes.models import ( + ConstantMetadata, + DeclarationExplanation, + DeclarationIdentity, + DeclarationRecord, + DeclarationReport, + EnumMetadata, + MacroMetadata, + MovePreview, + ResourceIdentity, + ScopeBrowseResult, + ScopeChainEntry, + ScopeCompleteness, + ScopeDiagnostic, + ScopeFolder, + ScopeListResult, + ScopeLookup, + ScopeReport, + ScopeReportFilters, + ScopeResourceReport, + ScopeSection, + SourceLocation, + VisibilityProvenance, +) +from sqlbuild.compiler.scopes.types import ( + NativeScopeBrowseRow, + NativeScopeDeclarationRow, + NativeScopeExplanationRow, + NativeScopeFailureRow, + NativeScopeFiltersRow, + NativeScopeListRow, + NativeScopeMoveRow, + NativeScopeRaisedRow, + NativeScopeReportRow, + NativeScopeSectionRow, + NativeScopeTargetRow, + NativeScopeValuesRow, + ScopeDiagnosticCode, + ScopeQueryTarget, +) + +_FAILURES: dict[str, type[Exception]] = { + "identity": InvalidQualifiedIdentityError, + "path": InvalidScopePathError, + "index": IndexError, +} + + +def build_scope_report( + *, + lookup: ScopeLookup, + target: ScopeQueryTarget | None, + at: str | PurePath | None, + directory: bool, + filters: ScopeReportFilters | None, +) -> ScopeReport: + """Query one existing target or prospective path; invalid index paths raise.""" + + selected: ScopeReportFilters = filters or ScopeReportFilters() + row, failure = _reports(lookup).report( + ( + None if target is None else target_row(target=target), + None if at is None else str(at), + directory, + filters_row(filters=selected), + ) + ) + return _report(lookup=lookup, row=_checked(row=row, failure=failure), filters=selected) + + +def explain_declaration( + *, + lookup: ScopeLookup, + declaration: str | DeclarationIdentity, + target: str | PurePath | ResourceIdentity | None, + at: str | PurePath | None, + directory: bool, +) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: + """Explain one qualified declaration in an optional resource or path context.""" + + row, failure = _reports(lookup).explain( + ( + ( + (qualified_row(identity=declaration), "") + if isinstance(declaration, DeclarationIdentity) + else (None, str(declaration)) + ), + None if target is None else target_row(target=target), + None if at is None else str(at), + directory, + ) + ) + explanation, diagnostics = _checked(row=row, failure=failure) + return _explanation(row=explanation), _diagnostics(lookup=lookup, rows=diagnostics) + + +def browse_folders( + *, + lookup: ScopeLookup, + folder: str, + target: str | PurePath | ResourceIdentity | None, + target_is_prospective: bool, +) -> ScopeBrowseResult: + """The direct child declaration-definition folders of `folder`.""" + + folder_text, folders, diagnostics, complete = _reports(lookup).browse( + ( + folder, + None if target is None else target_row(target=target), + target_is_prospective, + None, + ) + ) + return _browse(lookup=lookup, row=(folder_text, folders, diagnostics, complete)) + + +def list_folder( + *, + lookup: ScopeLookup, + folder: str, + filters: ScopeReportFilters | None, + target: str | PurePath | ResourceIdentity | None, + target_is_prospective: bool, +) -> ScopeListResult: + """Declarations recursively beneath one definition folder.""" + + row, failure = _reports(lookup).list( + ( + folder, + None if target is None else target_row(target=target), + target_is_prospective, + None if filters is None else filters_row(filters=filters), + ) + ) + return _list(lookup=lookup, row=_checked(row=row, failure=failure)) + + +def build_move_preview( + *, lookup: ScopeLookup, resource: str | ResourceIdentity, destination: str | PurePath +) -> tuple[MovePreview | None, tuple[ScopeDiagnostic, ...]]: + """Visibility deltas of a resource move, without mutating the lookup.""" + + row, failure = _reports(lookup).move_preview((target_row(target=resource), str(destination))) + preview, diagnostics = _checked(row=row, failure=failure) + return ( + None if preview is None else _move(row=preview), + _diagnostics(lookup=lookup, rows=diagnostics), + ) + + +def target_row(*, target: ScopeQueryTarget) -> NativeScopeTargetRow: + """A query target as the native engine reads it.""" + + if isinstance(target, (ResourceIdentity, DeclarationIdentity)): + return qualified_row(identity=target), "", False + return None, str(target), isinstance(target, PurePath) + + +def filters_row(*, filters: ScopeReportFilters) -> NativeScopeFiltersRow: + """Report filters as the native engine reads them.""" + + return ( + list(filters.sections), + filters.include_nearby, + filters.defined_under, + [str(kind) for kind in filters.kinds], + filters.glob, + filters.used_only, + filters.dependency_depth, + filters.sort, + filters.cursor, + filters.page_size, + filters.nearby_depth, + filters.globals, + ) + + +def values_row(*, record: DeclarationRecord) -> NativeScopeValuesRow: + """One declaration's value metadata as the native engine reads it.""" + + macro: MacroMetadata | None = record.macro + enum: EnumMetadata | None = record.enum + constant: ConstantMetadata | None = record.constant + return ( + None + if macro is None + else ( + list(macro.parameters), + [qualified_row(identity=item) for item in macro.dependencies], + macro.source_digest, + ), + None if enum is None else ([member.name for member in enum.members], enum.scalar_type), + None + if constant is None + else ( + constant.logical_type, + constant.collection_kind, + constant.item_count, + constant.nullable, + constant.render_as, + ), + ) + + +def _reports(lookup: ScopeLookup) -> _native.NativeScopeReports: + completeness: ScopeCompleteness = lookup.index.completeness + return _native.build_native_scope_reports( + cast(_native.NativeScopeLookup, lookup.native), + ( + [values_row(record=record) for record in lookup.index.declarations], + ( + completeness.discovery, + completeness.static_visibility, + completeness.runtime_usage, + completeness.relationships, + completeness.placement, + completeness.promotion_impact, + ), + ), + ) + + +def _checked[Row](*, row: Row | None, failure: NativeScopeFailureRow) -> Row: + if failure is not None: + kind, message = failure + raise _FAILURES[kind](message) + return cast("Row", row) + + +def _diagnostics( + *, lookup: ScopeLookup, rows: list[NativeScopeRaisedRow] +) -> tuple[ScopeDiagnostic, ...]: + diagnostics: list[ScopeDiagnostic] = [] + for position, raised in rows: + if position is not None: + diagnostics.append(lookup.index.diagnostics[position]) + continue + code, message, path = cast("tuple[str, str, str | None]", raised) + diagnostics.append(ScopeDiagnostic(ScopeDiagnosticCode(code), message, path=path)) + return tuple(diagnostics) + + +def _declaration(row: NativeScopeDeclarationRow) -> DeclarationReport: + ( + (identity, kind, name, owner, (path, line, column), scope, owning_path), + (visibility, inaccessible_reason, metadata), + (consumers, dependencies, grants, required_scope, required_path, promotion_impact), + ) = row + return DeclarationReport( + identity=identity, + kind=kind, + name=name, + owner=owner, + definition=SourceLocation(path, line, column), + scope=scope, + owning_path=owning_path, + visibility=None if visibility is None else VisibilityProvenance(*visibility), + inaccessible_reason=inaccessible_reason, + metadata=tuple(metadata), + consumers=tuple(consumers), + dependencies=tuple(dependencies), + grants=tuple(grants), + required_scope=required_scope, + required_path=required_path, + promotion_impact=tuple(promotion_impact), + ) + + +def _declarations(rows: list[NativeScopeDeclarationRow]) -> tuple[DeclarationReport, ...]: + return tuple(_declaration(row) for row in rows) + + +def _section(row: NativeScopeSectionRow) -> ScopeSection: + return ScopeSection(*row) + + +def _explanation(*, row: NativeScopeExplanationRow) -> DeclarationExplanation: + declaration, complete = row + return DeclarationExplanation( + None if declaration is None else _declaration(declaration), complete + ) + + +def _move(*, row: NativeScopeMoveRow) -> MovePreview: + resource, destination, root, retained, gained, lost, invalidated, private, related, done = row + return MovePreview( + resource=resource, + destination=destination, + new_ownership_root=root, + retained=_declarations(retained), + gained=_declarations(gained), + lost=_declarations(lost), + invalidated_usages=tuple(invalidated), + private_retained=_declarations(private), + relationship_retained=_declarations(related), + complete=done, + ) + + +def _report( + *, + lookup: ScopeLookup, + row: NativeScopeReportRow[NativeScopeRaisedRow], + filters: ScopeReportFilters, +) -> ScopeReport: + ( + resource, + chain, + available, + used, + relationship_scope, + nearby, + _filters, + sections, + explanation, + move, + diagnostics, + complete, + ) = row + return ScopeReport( + resource=ScopeResourceReport(*resource), + scope_chain=tuple(ScopeChainEntry(*entry) for entry in chain), + available=_declarations(available), + used=_declarations(used), + relationship_scope=_declarations(relationship_scope), + nearby_unavailable=_declarations(nearby), + filters=filters, + sections=tuple(_section(section) for section in sections), + explanation=None if explanation is None else _explanation(row=explanation), + move_preview=None if move is None else _move(row=move), + diagnostics=_diagnostics(lookup=lookup, rows=diagnostics), + complete=complete, + ) + + +def _browse( + *, lookup: ScopeLookup, row: NativeScopeBrowseRow[NativeScopeRaisedRow] +) -> ScopeBrowseResult: + folder, folders, diagnostics, complete = row + return ScopeBrowseResult( + folder, + tuple( + ScopeFolder(path, name, descendants, children, used, tuple(kinds)) + for path, name, descendants, children, used, kinds in folders + ), + _diagnostics(lookup=lookup, rows=diagnostics), + complete, + ) + + +def _list(*, lookup: ScopeLookup, row: NativeScopeListRow[NativeScopeRaisedRow]) -> ScopeListResult: + folder, declarations, section, diagnostics = row + return ScopeListResult( + folder, + _declarations(declarations), + _section(section), + _diagnostics(lookup=lookup, rows=diagnostics), + ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/visibility.py b/src/sqlbuild/compiler/scopes/_helpers/visibility.py index facd691eb2..384014ca31 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/visibility.py +++ b/src/sqlbuild/compiler/scopes/_helpers/visibility.py @@ -1,24 +1,19 @@ -"""Canonical record-level declaration visibility resolution.""" +"""Declaration visibility over a scope lookup, resolved by the native scope lookup.""" from __future__ import annotations -from functools import lru_cache from pathlib import PurePath -from types import MappingProxyType +from typing import cast -from sqlbuild.compiler.scopes._helpers.identities import parse_identity +import sqlbuild._native as _native from sqlbuild.compiler.scopes._helpers.paths import normalize_path -from sqlbuild.compiler.scopes.constants import ( - CURRENT_PATH_COMPONENT, - PATH_SEPARATOR, - QUALIFIED_IDENTITY_SEPARATOR, -) +from sqlbuild.compiler.scopes._helpers.placement import qualified_row, root_row +from sqlbuild.compiler.scopes.constants import CURRENT_PATH_COMPONENT, PATH_SEPARATOR +from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError, InvalidScopePathError from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, DeclarationRecord, DeclarationVisibility, - DeclarationVisibilityIndex, - GrantRecord, InaccessibleRecord, OwnershipRoot, ResourceIdentity, @@ -29,11 +24,11 @@ VisibilityResolution, ) from sqlbuild.compiler.scopes.types import ( - GrantKind, InaccessibleReason, + NativeResolvedRows, + NativeResourceRecordRow, OwnershipRootKind, ResourceKind, - ScopeKind, VisibilityReason, ) @@ -43,41 +38,49 @@ def query_target( *, lookup: ScopeLookup, target: ResourceIdentity | DeclarationIdentity | str | PurePath ) -> ScopeTargetQuery: - if isinstance(target, ResourceIdentity): - return ScopeTargetQuery(target, lookup.resources.get(target, ())) - if isinstance(target, DeclarationIdentity): - return ScopeTargetQuery( - target, - declaration_matches=lookup.declarations.get(target, ()), - ) - raw: str = str(target) - if QUALIFIED_IDENTITY_SEPARATOR in raw: - identity: ResourceIdentity | DeclarationIdentity = parse_identity(value=raw) - if isinstance(identity, ResourceIdentity): - return ScopeTargetQuery(raw, lookup.resources.get(identity, ())) + """The resources or declarations `target` names; invalid text raises the scope error.""" + + identity: bool = isinstance(target, (ResourceIdentity, DeclarationIdentity)) + raw: str = "" if identity else str(target) + declarations, positions, identity_error, path_error = _lookup(lookup).target( + qualified_row(identity=cast("ResourceIdentity | DeclarationIdentity", target)) + if identity + else None, + raw, + ) + if identity_error is not None: + raise InvalidQualifiedIdentityError(identity_error) + if path_error is not None: + raise InvalidScopePathError(path_error) + value: str | ResourceIdentity | DeclarationIdentity = ( + cast("ResourceIdentity | DeclarationIdentity", target) if identity else raw + ) + if declarations: return ScopeTargetQuery( - raw, - declaration_matches=lookup.declarations.get(identity, ()), + value, + declaration_matches=tuple(lookup.index.declarations[item] for item in positions), ) - return ScopeTargetQuery(raw, lookup.resources_by_path.get(normalize_path(path=raw), ())) + return ScopeTargetQuery(value, tuple(lookup.index.resources[item] for item in positions)) def resolve_visibility( *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath ) -> VisibilityResolution: - query, visible, inaccessible_groups = _classify_target(lookup=lookup, target=target) - declarations: tuple[DeclarationRecord, ...] = lookup.index.declarations + """Every visible fact and inaccessible declaration, with reasons, of each matched resource.""" + + query: ScopeTargetQuery = query_target(lookup=lookup, target=target) + visible: list[VisibilityRecord] = [] inaccessible: list[InaccessibleRecord] = [] - resource: ResourceRecord - positions: list[int] - for resource, positions in inaccessible_groups: + for resource in query.matches: + resource_visible, resource_inaccessible = _resolve(lookup=lookup, resource=resource) + visible.extend(resource_visible) inaccessible.extend( InaccessibleRecord( resource.identity, - declarations[position].identity, - _inaccessible_reason(resource=resource, declaration=declarations[position]), + lookup.index.declarations[position].identity, + InaccessibleReason(reason), ) - for position in positions + for position, reason in resource_inaccessible ) return VisibilityResolution(query, tuple(visible), tuple(inaccessible)) @@ -87,164 +90,16 @@ def resolve_declaration_visibility( ) -> DeclarationVisibility: """Resolve visible facts and inaccessible identities without per-pair inaccessible reasons.""" - query, visible, inaccessible_groups = _classify_target(lookup=lookup, target=target) - identities: tuple[DeclarationIdentity, ...] = lookup.visibility_index.identities - inaccessible: list[DeclarationIdentity] = [] - positions: list[int] - for _resource, positions in inaccessible_groups: - inaccessible.extend(identities[position] for position in positions) - return DeclarationVisibility(query, tuple(visible), tuple(inaccessible)) - - -def _classify_target( - *, lookup: ScopeLookup, target: ResourceIdentity | str | PurePath -) -> tuple[ScopeTargetQuery, list[VisibilityRecord], list[tuple[ResourceRecord, list[int]]]]: - """Return the query, ordered visible records, and inaccessible positions per match.""" - query: ScopeTargetQuery = query_target(lookup=lookup, target=target) visible: list[VisibilityRecord] = [] - inaccessible_groups: list[tuple[ResourceRecord, list[int]]] = [] - resource: ResourceRecord + inaccessible: list[DeclarationIdentity] = [] for resource in query.matches: - resource_visible, positions = _classify_resource(lookup=lookup, resource=resource) + resource_visible, resource_inaccessible = _resolve(lookup=lookup, resource=resource) visible.extend(resource_visible) - inaccessible_groups.append((resource, positions)) - return query, visible, inaccessible_groups - - -def _classify_resource( - *, lookup: ScopeLookup, resource: ResourceRecord -) -> tuple[list[VisibilityRecord], list[int]]: - """Return ordered visible records and the declaration positions left inaccessible.""" - - index: DeclarationVisibilityIndex = lookup.visibility_index - positive: list[tuple[int, VisibilityReason]] = _visible_positions( - index=index, resource=resource - ) - grants: tuple[GrantRecord, ...] = lookup.grants_by_resource.get(resource.identity, ()) - if not grants: - visible_positions: set[int] = {position for position, _reason in positive} - return ( - [ - VisibilityRecord(resource.identity, index.identities[position], reason) - for position, reason in positive - ], - [ - position - for position in range(len(index.identities)) - if position not in visible_positions - ], - ) - reasons: dict[int, VisibilityReason] = dict(positive) - grants_by_declaration: dict[DeclarationIdentity, list[GrantRecord]] = {} - for grant in grants: - grants_by_declaration.setdefault(grant.declaration, []).append(grant) - visible: list[VisibilityRecord] = [] - inaccessible: list[int] = [] - position: int - identity: DeclarationIdentity - for position, identity in enumerate(index.identities): - reason: VisibilityReason | None = reasons.get(position) - if reason is not None: - visible.append(VisibilityRecord(resource.identity, identity, reason)) - declaration_grants: list[GrantRecord] | None = grants_by_declaration.get(identity) - if declaration_grants is not None: - visible.extend( - VisibilityRecord( - resource.identity, - identity, - ( - VisibilityReason.TESTED_MACRO - if grant.kind is GrantKind.TESTED_MACRO - else VisibilityReason.EXPECTED_MODEL - ), - grant.through, - ) - for grant in declaration_grants - ) - elif reason is None: - inaccessible.append(position) - return visible, inaccessible - - -def build_visibility_index( - *, declarations: tuple[DeclarationRecord, ...] -) -> DeclarationVisibilityIndex: - """Group canonical declaration positions by the scope facts that make them visible.""" - - global_positions: list[int] = [] - private: dict[ResourceIdentity, list[int]] = {} - local: dict[str, list[int]] = {} - inherited: dict[str, list[int]] = {} - position: int - declaration: DeclarationRecord - for position, declaration in enumerate(declarations): - if declaration.scope is ScopeKind.GLOBAL: - global_positions.append(position) - elif declaration.scope is ScopeKind.PRIVATE and declaration.identity.owner is not None: - private.setdefault(declaration.identity.owner, []).append(position) - elif declaration.scope is ScopeKind.LOCAL: - local.setdefault(_declaration_owner(declaration), []).append(position) - elif declaration.scope is ScopeKind.INHERITED: - inherited.setdefault(_declaration_owner(declaration), []).append(position) - return DeclarationVisibilityIndex( - identities=tuple(declaration.identity for declaration in declarations), - global_positions=tuple(global_positions), - private_positions=MappingProxyType( - {owner: tuple(positions) for owner, positions in private.items()} - ), - local_positions=MappingProxyType( - {owner: tuple(positions) for owner, positions in local.items()} - ), - inherited_positions=MappingProxyType( - {owner: tuple(positions) for owner, positions in inherited.items()} - ), - ) - - -def _visible_positions( - *, index: DeclarationVisibilityIndex, resource: ResourceRecord -) -> list[tuple[int, VisibilityReason]]: - """Return declaration positions visible to a resource, in declaration order.""" - - positive: list[tuple[int, VisibilityReason]] = [ - (position, VisibilityReason.GLOBAL) for position in index.global_positions - ] - positive.extend( - (position, VisibilityReason.PRIVATE_OWNER) - for position in index.private_positions.get(resource.identity, ()) - ) - if index.local_positions or index.inherited_positions: - parent: str = _parent(resource.path) - positive.extend( - (position, VisibilityReason.LOCAL_OWNER) - for position in index.local_positions.get(parent, ()) + inaccessible.extend( + lookup.index.declarations[position].identity for position, _ in resource_inaccessible ) - ancestor: str - for ancestor in _canonical_ancestors(parent): - positive.extend( - (position, VisibilityReason.INHERITED_ANCESTOR) - for position in index.inherited_positions.get(ancestor, ()) - ) - positive.sort() - return positive - - -def _declaration_owner(declaration: DeclarationRecord) -> str: - return _canonical_owner(declaration.owning_path or CURRENT_PATH_COMPONENT) - - -@lru_cache(maxsize=65_536) -def _canonical_ancestors(path: str) -> tuple[str, ...]: - """Return every owner that ``_is_canonical_descendant`` accepts as an ancestor of ``path``.""" - - prefixes: list[str] = [CURRENT_PATH_COMPONENT] - index: int = path.find(PATH_SEPARATOR) - while index >= 0: - prefixes.append(path[:index]) - index = path.find(PATH_SEPARATOR, index + 1) - prefixes.append(path) - return tuple(dict.fromkeys(prefixes)) + return DeclarationVisibility(query, tuple(visible), tuple(inaccessible)) def resolve_path_visibility( @@ -262,16 +117,14 @@ def resolve_path_visibility( resource_kind=ResourceKind.MODEL, ), ) - visible: list[DeclarationRecord] = [] - inaccessible: list[DeclarationRecord] = [] - for declaration in lookup.index.declarations: - target: list[DeclarationRecord] = ( - visible - if _visibility_reason(resource=resource, declaration=declaration) is not None - else inaccessible - ) - target.append(declaration) - return tuple(visible), tuple(inaccessible) + visible, inaccessible, error = _resolved_rows(lookup=lookup, resource=resource) + if error is not None: + raise InvalidScopePathError(error) + declarations: tuple[DeclarationRecord, ...] = lookup.index.declarations + return ( + tuple(declarations[position] for position, _, _ in visible), + tuple(declarations[position] for position, _ in inaccessible), + ) def declaration_lexical_path(*, record: DeclarationRecord) -> str: @@ -280,78 +133,38 @@ def declaration_lexical_path(*, record: DeclarationRecord) -> str: return f"{record.owning_path or CURRENT_PATH_COMPONENT}{PATH_SEPARATOR}{_DECLARATION_STEM}" -def path_visibility_reason( - *, declaration: DeclarationRecord, path: str | PurePath -) -> VisibilityReason | None: - """Return why a public declaration is visible from an authored path, if it is.""" - - normalized_path: str = normalize_path(path=path) - return _visibility_reason( - resource=ResourceRecord( - identity=ResourceIdentity(ResourceKind.MODEL, f""), - path=normalized_path, - ownership_root=OwnershipRoot(path=CURRENT_PATH_COMPONENT), - ), - declaration=declaration, +def _resolve( + *, lookup: ScopeLookup, resource: ResourceRecord +) -> tuple[list[VisibilityRecord], list[tuple[int, str]]]: + visible, inaccessible, error = _resolved_rows(lookup=lookup, resource=resource) + if error is not None: + raise InvalidScopePathError(error) + identities: tuple[DeclarationIdentity, ...] = lookup.visibility_index.identities + return ( + [ + VisibilityRecord( + resource.identity, + identities[position], + VisibilityReason(reason), + None if grant is None else lookup.index.grants[grant].through, + ) + for position, reason, grant in visible + ], + inaccessible, ) -def _visibility_reason( - *, resource: ResourceRecord, declaration: DeclarationRecord -) -> VisibilityReason | None: - if declaration.scope is ScopeKind.GLOBAL: - return VisibilityReason.GLOBAL - if declaration.scope is ScopeKind.PRIVATE: - return ( - VisibilityReason.PRIVATE_OWNER - if declaration.identity.owner == resource.identity - else None - ) - owner: str = _canonical_owner(declaration.owning_path or CURRENT_PATH_COMPONENT) - parent: str = _parent(resource.path) - if declaration.scope is ScopeKind.LOCAL: - return VisibilityReason.LOCAL_OWNER if parent == owner else None - if declaration.scope is ScopeKind.INHERITED and _is_canonical_descendant( - path=parent, ancestor=owner - ): - return VisibilityReason.INHERITED_ANCESTOR - return None - - -def _inaccessible_reason( - *, resource: ResourceRecord, declaration: DeclarationRecord -) -> InaccessibleReason: - if declaration.scope is ScopeKind.PRIVATE: - return InaccessibleReason.PRIVATE_OWNER - if declaration.scope is ScopeKind.LOCAL: - return InaccessibleReason.LOCAL_BOUNDARY - owner: str = _canonical_owner(declaration.owning_path or CURRENT_PATH_COMPONENT) - parent: str = _parent(resource.path) - if _is_canonical_descendant(path=owner, ancestor=parent): - return InaccessibleReason.DESCENDANT_SCOPE - if declaration.ownership_root.path == resource.ownership_root.path: - return InaccessibleReason.SIBLING_SCOPE - return InaccessibleReason.UNRELATED_SCOPE - - -@lru_cache(maxsize=65_536) -def _parent(path: str) -> str: - normalized: str = normalize_path(path=path) - return ( - normalized.rsplit(PATH_SEPARATOR, maxsplit=1)[0] - if PATH_SEPARATOR in normalized - else CURRENT_PATH_COMPONENT - ) +def _resolved_rows(*, lookup: ScopeLookup, resource: ResourceRecord) -> NativeResolvedRows: + return _lookup(lookup).resolve(_resource_row(resource=resource)) -def _is_canonical_descendant(*, path: str, ancestor: str) -> bool: +def _resource_row(*, resource: ResourceRecord) -> NativeResourceRecordRow: return ( - ancestor == CURRENT_PATH_COMPONENT - or path == ancestor - or path.startswith(f"{ancestor}{PATH_SEPARATOR}") + (resource.identity.kind.value, resource.identity.name), + resource.path, + root_row(root=resource.ownership_root), ) -@lru_cache(maxsize=65_536) -def _canonical_owner(path: str) -> str: - return normalize_path(path=path) +def _lookup(lookup: ScopeLookup) -> _native.NativeScopeLookup: + return cast(_native.NativeScopeLookup, lookup.native) diff --git a/src/sqlbuild/compiler/scopes/classes/native_scope_index.py b/src/sqlbuild/compiler/scopes/classes/native_scope_index.py index f7b8bea4c3..d2917488f4 100644 --- a/src/sqlbuild/compiler/scopes/classes/native_scope_index.py +++ b/src/sqlbuild/compiler/scopes/classes/native_scope_index.py @@ -2,20 +2,19 @@ from __future__ import annotations -from collections.abc import Callable, Hashable, Sequence from dataclasses import replace -from types import MappingProxyType +from pathlib import Path import sqlbuild._native as _native +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestFile +from sqlbuild.compiler.scopes._helpers.lookup import build_lookup from sqlbuild.compiler.scopes.classes.native_scope_rows import NativeScopeRows from sqlbuild.compiler.scopes.exceptions import UnindexableScopeError from sqlbuild.compiler.scopes.models import ( DeclarationRecord, - DeclarationVisibilityIndex, GrantRecord, NativeDeclarationValues, OwnershipRoot, - RelationshipFact, ResourceIdentity, ResourceRecord, ScopeCompleteness, @@ -31,8 +30,8 @@ ScopeDiagnosticCode, UsageKind, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -type _Groups = tuple[list[list[int]], bool] type _IndexRows = tuple[ list[str], list[tuple[str, str, str, str | None, str | None]], @@ -43,20 +42,6 @@ bool, ] type _GrantRows = list[tuple[str, str, int, str | int, str]] -type _LookupRows = tuple[ - list[int], - list[int], - list[int], - list[int], - tuple[_Groups, _Groups, _Groups, _Groups, _Groups, _Groups], - tuple[ - list[int], - list[tuple[tuple[str, str], list[int]]], - list[tuple[str, list[int]]], - list[tuple[str, list[int]]], - ], -] - _RESOURCE_ROOTS: dict[ResourceKind, str] = { ResourceKind.MODEL: "models", ResourceKind.TEST: "tests/unit", @@ -133,20 +118,27 @@ def __init__(self, *, native: _native.NativeScopeIndex, rows: NativeScopeRows) - completeness=_STATIC_COMPLETENESS, ) - def grant(self, facts: Sequence[RelationshipFact]) -> None: - """Attach the relationship grants of `facts`.""" - - rows: _GrantRows | None = self._native.grant( + def grant_relationships( + self, + *, + discovered_inputs: DiscoveredProjectInputs, + syntax: SqlLexicalSyntax, + scan_store: tuple[_native.SqlTestScanStore | None, str], + ) -> tuple[tuple[Path, str], ...]: + """Grant every test's and scenario's relationships; return each extraction fault.""" + + store, algorithm = scan_store + rows, faults = self._native.grant_relationships( + [_test_file_row(test_file) for test_file in discovered_inputs.test_files], [ - ( - fact.resource.kind.value, - fact.resource.name, - list(fact.expected_models), - list(fact.called_macros), - list(fact.tested_macros), - ) - for fact in facts - ] + (scenario.name, str(scenario.relative_path), scenario.sql_body) + for scenario in discovered_inputs.scenario_files + ], + { + "syntax": syntax.native_mapping, + "store": store, + "scan_key": (algorithm, syntax.cache_key), + }, ) if rows is None: raise UnindexableScopeError @@ -166,6 +158,7 @@ def grant(self, facts: Sequence[RelationshipFact]) -> None: ) for kind, name, declaration, through, grant_kind in rows ] + return tuple((Path(path), message) for path, message in faults) def index_with_relationships(self) -> ScopeIndex: """Return the index with its grants attached and relationships complete.""" @@ -177,83 +170,19 @@ def index_with_relationships(self) -> ScopeIndex: ) def lookup(self, *, index: ScopeIndex) -> ScopeLookup: - """Return Python's `build_lookup(index)` for `index_with_relationships()`.""" + """The native scope lookup of `index_with_relationships()`.""" - rows: _LookupRows | None = self._native.lookup() - if rows is None: - raise UnindexableScopeError - resource_positions, declaration_positions, usage_positions, grant_positions = rows[:4] - groups: tuple[_Groups, _Groups, _Groups, _Groups, _Groups, _Groups] = rows[4] - global_positions, private_positions, local_positions, inherited_positions = rows[5] - declarations: tuple[DeclarationRecord, ...] = tuple( - self._declarations[position] for position in declaration_positions - ) - canonical_index: ScopeIndex = ScopeIndex( - ownership_roots=tuple( - sorted( - index.ownership_roots, - key=lambda item: ( - item.path, - item.resource_kind.value if item.resource_kind is not None else "", - ), - ) - ), - resources=tuple(self._resources[position] for position in resource_positions), - declarations=declarations, - usages=tuple(self._usages[position] for position in usage_positions), - grants=tuple(self._grants[position] for position in grant_positions), - visibility=(), - inaccessible=(), - diagnostics=tuple( - sorted( - index.diagnostics, - key=lambda item: ( - item.path or "", - item.line or 0, - item.column or 0, - item.code.value, - ), - ) - ), - completeness=index.completeness, - ) - return ScopeLookup( - index=canonical_index, - resources=_mapping( - groups=groups[0], records=self._resources, key=lambda item: item.identity - ), - resources_by_path=_mapping( - groups=groups[1], records=self._resources, key=lambda item: item.path - ), - declarations=_mapping( - groups=groups[2], records=self._declarations, key=lambda item: item.identity - ), - usages_by_consumer=_mapping( - groups=groups[3], records=self._usages, key=lambda item: item.consumer - ), - usages_by_declaration=_mapping( - groups=groups[4], records=self._usages, key=lambda item: item.declaration - ), - grants_by_resource=_mapping( - groups=groups[5], records=self._grants, key=lambda item: item.resource - ), - visibility_index=DeclarationVisibilityIndex( - identities=tuple(declaration.identity for declaration in declarations), - global_positions=tuple(global_positions), - private_positions=MappingProxyType( - { - ResourceIdentity(ResourceKind(kind), name): tuple(positions) - for (kind, name), positions in private_positions - } - ), - local_positions=MappingProxyType( - {owner: tuple(positions) for owner, positions in local_positions} - ), - inherited_positions=MappingProxyType( - {owner: tuple(positions) for owner, positions in inherited_positions} - ), - ), - ) + return build_lookup(index=index) + + +def _test_file_row( + test_file: DiscoveredSqlTestFile, +) -> tuple[str, list[tuple[str, str, str]]]: + stem: str = test_file.relative_path.stem + return ( + str(test_file.relative_path), + [(block.name or stem, block.sql_body, block.mode.value) for block in test_file.blocks], + ) def _resource_root(*, identity: ResourceIdentity, root: str) -> OwnershipRoot: @@ -283,16 +212,3 @@ def _declaration_record( enum=values.enum, constant=values.constant, ) - - -def _mapping[Record, Key: Hashable]( - *, groups: _Groups, records: Sequence[Record], key: Callable[[Record], Key] -) -> MappingProxyType[Key, tuple[Record, ...]]: - members, repr_ordered = groups - items: list[tuple[Key, tuple[Record, ...]]] = [] - for group in members: - grouped: tuple[Record, ...] = tuple(records[member] for member in group) - items.append((key(grouped[0]), grouped)) - if not repr_ordered: - items.sort(key=lambda item: repr(item[0])) - return MappingProxyType(dict(items)) diff --git a/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py b/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py index 3e5d07295d..c488587ab5 100644 --- a/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py +++ b/src/sqlbuild/compiler/scopes/classes/native_scope_rows.py @@ -2,10 +2,13 @@ from __future__ import annotations +import hashlib +import inspect from collections.abc import Mapping from pathlib import Path +from types import CodeType -from sqlbuild.compiler.compile.models import LoadedMacro +from sqlbuild.compiler.compile.models import LoadedMacro, StaticMacroExport from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, DiscoveredAuditFile, @@ -16,15 +19,13 @@ DiscoveredSqlHookFile, EnumDeclaration, ) -from sqlbuild.compiler.scopes._helpers.builder import ( - constant_metadata, - enum_metadata, - macro_line, - macro_metadata, -) from sqlbuild.compiler.scopes._helpers.paths import normalize_path from sqlbuild.compiler.scopes.models import ( + ConstantMetadata, DeclarationIdentity, + EnumMemberMetadata, + EnumMetadata, + MacroMetadata, NativeDeclarationValues, ResourceIdentity, ) @@ -36,6 +37,7 @@ ResourceKind, ScopeKind, ) +from sqlbuild.sql_values.types import SqlValueKind _KIND_ROOT: str = "kind" _SEED_ROOT: str = "seed" @@ -44,6 +46,9 @@ _MACRO_FALLBACK: str = "macros" _ENUM_FALLBACK: str = "enums" _CONSTANT_FALLBACK: str = "constants" +_COLLECTION_KINDS: frozenset[SqlValueKind] = frozenset( + {SqlValueKind.LIST, SqlValueKind.SET, SqlValueKind.OBJECT} +) class NativeScopeRows: @@ -54,6 +59,7 @@ def __init__( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro], + static_macros: tuple[StaticMacroExport, ...] = (), ) -> None: self.resources: list[NativeResourceRow] = [] self.resource_identities: list[ResourceIdentity] = [] @@ -65,6 +71,7 @@ def __init__( self._add_declaration_files(discovered_inputs=discovered_inputs) self._add_named_declarations(discovered_inputs=discovered_inputs) self._add_macros(discovered_inputs=discovered_inputs, loaded_macros=loaded_macros) + self._add_static_macros(discovered_inputs=discovered_inputs, static_macros=static_macros) def _add_resource(self, *, kind: ResourceKind, name: str, path: Path, root: str) -> None: self.resources.append((kind.value, name, str(path), root)) @@ -129,14 +136,14 @@ def _add_private( identity=DeclarationIdentity(DeclarationKind.ENUM, declaration.name, owner), line=1, scope=ScopeKind.PRIVATE, - enum=enum_metadata(declaration), + enum=_enum_metadata(declaration), ) if isinstance(declaration, EnumDeclaration) else NativeDeclarationValues( identity=DeclarationIdentity(DeclarationKind.CONSTANT, declaration.name, owner), line=1, scope=ScopeKind.PRIVATE, - constant=constant_metadata(declaration), + constant=_constant_metadata(declaration), ) ) self._add_declaration( @@ -204,7 +211,7 @@ def _add_declaration_files(self, *, discovered_inputs: DiscoveredProjectInputs) identity=DeclarationIdentity(DeclarationKind.ENUM, enum.name), line=1, scope=enum_file.scope_kind, - enum=enum_metadata(enum), + enum=_enum_metadata(enum), ), path=enum_file.relative_path, ownership_root=enum_file.ownership_root, @@ -218,7 +225,7 @@ def _add_declaration_files(self, *, discovered_inputs: DiscoveredProjectInputs) identity=DeclarationIdentity(DeclarationKind.CONSTANT, constant.name), line=1, scope=constant_file.scope_kind, - constant=constant_metadata(constant), + constant=_constant_metadata(constant), ), path=constant_file.relative_path, ownership_root=constant_file.ownership_root, @@ -303,9 +310,9 @@ def _add_macros( self._add_declaration( values=NativeDeclarationValues( identity=DeclarationIdentity(DeclarationKind.MACRO, loaded.name), - line=macro_line(loaded), + line=_macro_line(loaded), scope=discovered.scope_kind if discovered is not None else ScopeKind.GLOBAL, - macro=macro_metadata(loaded), + macro=_macro_metadata(loaded), dependencies=loaded.dependencies, ), path=loaded.relative_path, @@ -314,6 +321,37 @@ def _add_macros( owning_path=discovered.owning_path if discovered is not None else None, ) + def _add_static_macros( + self, + *, + discovered_inputs: DiscoveredProjectInputs, + static_macros: tuple[StaticMacroExport, ...], + ) -> None: + macro_files: dict[str, DiscoveredMacroFile] = { + normalize_path(path=item.relative_path): item for item in discovered_inputs.macro_files + } + for export in static_macros: + discovered: DiscoveredMacroFile | None = macro_files.get( + normalize_path(path=export.relative_path) + ) + self._add_declaration( + values=NativeDeclarationValues( + identity=DeclarationIdentity(DeclarationKind.MACRO, export.name), + line=export.line, + scope=discovered.scope_kind if discovered is not None else ScopeKind.GLOBAL, + macro=MacroMetadata( + parameters=export.parameters, + dependencies=export.dependencies, + source_digest=export.source_digest, + ), + dependencies=export.dependencies, + ), + path=export.relative_path, + ownership_root=discovered.ownership_root if discovered is not None else None, + root_fallback=_MACRO_FALLBACK, + owning_path=discovered.owning_path if discovered is not None else None, + ) + def _identity_row(identity: ResourceIdentity | None) -> NativeIdentityRow | None: return None if identity is None else (identity.kind.value, identity.name) @@ -321,3 +359,44 @@ def _identity_row(identity: ResourceIdentity | None) -> NativeIdentityRow | None def _name_or_stem(*, value: object, path: Path) -> str: return value if isinstance(value, str) and value else path.stem + + +def _enum_metadata(declaration: EnumDeclaration) -> EnumMetadata: + """Return the member names and scalar type of one enum.""" + + return EnumMetadata( + members=tuple(EnumMemberMetadata(item.name) for item in declaration.members), + scalar_type=declaration.scalar_type, + ) + + +def _constant_metadata(declaration: ConstantDeclaration) -> ConstantMetadata: + """Return the logical type and collection shape of one constant.""" + + is_collection: bool = declaration.value.kind in _COLLECTION_KINDS + payload: object = declaration.value.value + item_count: int | None = len(payload) if is_collection and isinstance(payload, tuple) else None + return ConstantMetadata( + logical_type=declaration.logical_type.display_name, + collection_kind=declaration.value.kind.value if is_collection else None, + item_count=item_count, + nullable=declaration.value.kind is SqlValueKind.NULL, + render_as=declaration.render_as.value if declaration.render_as is not None else None, + ) + + +def _macro_metadata(loaded: LoadedMacro) -> MacroMetadata: + """Return the parameters, dependencies and source digest of one loaded macro.""" + + return MacroMetadata( + parameters=tuple(inspect.signature(loaded.function).parameters), + dependencies=loaded.dependencies, + source_digest=hashlib.sha256(loaded.raw_source.encode()).hexdigest(), + ) + + +def _macro_line(loaded: LoadedMacro) -> int: + """Return the first line of a loaded macro's function, or 1 without code.""" + + code: object = getattr(loaded.function, "__code__", None) + return code.co_firstlineno if isinstance(code, CodeType) else 1 diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index 02d72e4e92..c00c1a6d13 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -5,35 +5,8 @@ from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind SCOPE_METADATA_SCHEMA_VERSION: int = 3 -SCOPE_CACHE_SCHEMA_VERSION: int = 3 -SCOPE_FINGERPRINT_ALGORITHM_VERSION: int = 2 SCOPE_CACHE_DIRECTORY_NAME: str = "declaration-scopes-v2" SCOPE_CACHE_FILENAME: str = "scope-index.json" -SCOPE_CACHE_MAX_BYTES: int = 16 * 1024 * 1024 -SCOPE_SOURCE_SUFFIXES: frozenset[str] = frozenset({".sql", ".yml", ".yaml", ".py"}) -SCOPE_MACRO_SUFFIX: str = ".py" -SCOPE_RELATIONSHIP_ROOTS: frozenset[tuple[str, str]] = frozenset( - {("tests", "unit"), ("tests", "scenarios")} -) -SCOPE_LOCAL_CONFIG_KEYS: frozenset[str] = frozenset( - {"adapter", "settings", "target", "targets", "vars"} -) -SCOPE_PROJECT_CONFIG_KEYS: frozenset[str] = frozenset( - { - "adapter", - "constants", - "default_target", - "defaults", - "path_defaults", - "scopes", - "settings", - "targets", - "vars", - } -) -SCOPE_TARGET_VARS_KEY: str = "vars" -SCOPE_TARGETS_KEY: str = "targets" -SCOPE_ENUM_MEMBER_FIELDS: frozenset[str] = frozenset({"name"}) DEFAULT_ENUM_MEMBER_PREVIEW: int = 20 QUALIFIED_IDENTITY_SEPARATOR: str = ":" PATH_SEPARATOR: str = "/" diff --git a/src/sqlbuild/compiler/scopes/exceptions.py b/src/sqlbuild/compiler/scopes/exceptions.py index bc31940329..739a0f3b82 100644 --- a/src/sqlbuild/compiler/scopes/exceptions.py +++ b/src/sqlbuild/compiler/scopes/exceptions.py @@ -39,10 +39,6 @@ def __init__(self) -> None: ) -class ScopeCacheDecodeError(ScopeError): - """Raised when a persistent scope-index payload is malformed.""" - - class ScopeValidationError(ScopeError): """Aggregate error raised only when strict scope validation is requested.""" diff --git a/src/sqlbuild/compiler/scopes/main/_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/_build_scope_index.py index c983da7e7e..c1a3187392 100644 --- a/src/sqlbuild/compiler/scopes/main/_build_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/_build_scope_index.py @@ -6,7 +6,7 @@ from sqlbuild.compiler.compile.models import LoadedMacro, StaticMacroExport from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes._helpers.builder import build_index +from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.models import ScopeIndex @@ -18,8 +18,8 @@ def build_scope_index( ) -> ScopeIndex: """Project discovery facts into a deterministic, partially valid scope index.""" - return build_index( + return open_native_scope_index( discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, + loaded_macros=loaded_macros or {}, static_macros=static_macros, - ) + ).index diff --git a/src/sqlbuild/compiler/scopes/main/_declaration_visibility.py b/src/sqlbuild/compiler/scopes/main/_declaration_visibility.py deleted file mode 100644 index 9c0a21904e..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_declaration_visibility.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Visibility of one declaration from an authored consumer path.""" - -from __future__ import annotations - -from pathlib import PurePath - -from sqlbuild.compiler.scopes._helpers.visibility import ( - declaration_lexical_path, - path_visibility_reason, -) -from sqlbuild.compiler.scopes.models import DeclarationRecord -from sqlbuild.compiler.scopes.types import VisibilityReason - - -def declaration_visibility( - *, declaration: DeclarationRecord, consumer: str | PurePath | DeclarationRecord -) -> VisibilityReason | None: - """Return why a declaration is visible from a resource path or a declaration's owner folder.""" - - path: str | PurePath = ( - declaration_lexical_path(record=consumer) - if isinstance(consumer, DeclarationRecord) - else consumer - ) - return path_visibility_reason(declaration=declaration, path=path) diff --git a/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py b/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py index b8b142b04f..f5cb4fd0af 100644 --- a/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/_get_placement_validated_scope_index.py @@ -2,13 +2,21 @@ from __future__ import annotations -from sqlbuild.compiler.scopes._helpers.placement import build_placement_validated_index -from sqlbuild.compiler.scopes.models import ScopeIndex +from sqlbuild.compiler.scopes._helpers.placement import ( + build_placement_validated_index, + merged_usage_index, +) +from sqlbuild.compiler.scopes.models import ScopeIndex, UsageRecord def get_placement_validated_scope_index( - *, index: ScopeIndex, enforce_placement: bool = True + *, + index: ScopeIndex, + enforce_placement: bool = True, + compile_usages: tuple[UsageRecord, ...] | None = None, ) -> ScopeIndex: - """Return complete deterministic usage and placement diagnostics.""" + """Return complete usage and placement diagnostics, merging any collected compile usages.""" + if compile_usages is not None: + index = merged_usage_index(index=index, collected=compile_usages) return build_placement_validated_index(index=index, enforce_placement=enforce_placement) diff --git a/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py b/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py deleted file mode 100644 index 5f91ab1e02..0000000000 --- a/src/sqlbuild/compiler/scopes/main/_native_expected_model_names.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Expected-model relationship names scanned natively for the native declaration scope.""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlbuild._native as _native -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def native_expected_model_names( - *, texts: Sequence[tuple[str, str]], scenario: bool, syntax: SqlLexicalSyntax -) -> list[tuple[str, ...] | str]: - """Return each `(sql, file label)` body's names or its error.""" - - if not texts: - return [] - outcomes: list[tuple[str | None, list[str]]] = _native.scope_expected_model_names( - list(texts), scenario, syntax.native_mapping - ) - return [outcome[0] if outcome[0] is not None else tuple(outcome[1]) for outcome in outcomes] diff --git a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py index fba02f1617..d0ad64a8ba 100644 --- a/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/_open_native_scope_index.py @@ -5,7 +5,7 @@ from collections.abc import Mapping import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import LoadedMacro +from sqlbuild.compiler.compile.models import LoadedMacro, StaticMacroExport from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex from sqlbuild.compiler.scopes.classes.native_scope_rows import NativeScopeRows @@ -13,12 +13,17 @@ def open_native_scope_index( - *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: Mapping[str, LoadedMacro] + *, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: Mapping[str, LoadedMacro], + static_macros: tuple[StaticMacroExport, ...] = (), ) -> NativeScopeIndex: """Return the natively built index; discovery's normalized paths always index.""" rows: NativeScopeRows = NativeScopeRows( - discovered_inputs=discovered_inputs, loaded_macros=loaded_macros + discovered_inputs=discovered_inputs, + loaded_macros=loaded_macros, + static_macros=static_macros, ) native: _native.NativeScopeIndex | None = _native.build_native_scope_index( rows.resources, rows.declarations diff --git a/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py b/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py index 9fc4f26eb1..379c9520d2 100644 --- a/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py +++ b/src/sqlbuild/compiler/scopes/main/browse_scope_folders.py @@ -2,7 +2,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.report_operations import browse_folders +from sqlbuild.compiler.scopes._helpers.reports import browse_folders from sqlbuild.compiler.scopes.models import ResourceIdentity, ScopeBrowseResult, ScopeLookup diff --git a/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py b/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py index 7c777e79e0..5484f70937 100644 --- a/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py +++ b/src/sqlbuild/compiler/scopes/main/explain_scope_declaration.py @@ -4,19 +4,13 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.report_query import ( - _prospective_resource, - explain_declaration, -) -from sqlbuild.compiler.scopes._helpers.visibility import query_target +from sqlbuild.compiler.scopes._helpers.reports import explain_declaration from sqlbuild.compiler.scopes.models import ( DeclarationExplanation, DeclarationIdentity, ResourceIdentity, - ResourceRecord, ScopeDiagnostic, ScopeLookup, - ScopeTargetQuery, ) @@ -30,16 +24,6 @@ def explain_scope_declaration( ) -> tuple[DeclarationExplanation, tuple[ScopeDiagnostic, ...]]: """Explain a qualified declaration in an optional resource/path context.""" - context: ResourceRecord | None = None - diagnostics: tuple[ScopeDiagnostic, ...] = () - if at is not None: - _resource, context, diagnostics = _prospective_resource( - lookup=lookup, at=at, directory=directory - ) - elif target is not None: - query: ScopeTargetQuery = query_target(lookup=lookup, target=target) - context = query.matches[0] if len(query.matches) == 1 else None - explanation, explanation_diagnostics = explain_declaration( - lookup=lookup, target=declaration, at=context + return explain_declaration( + lookup=lookup, declaration=declaration, target=target, at=at, directory=directory ) - return explanation, (*diagnostics, *explanation_diagnostics) diff --git a/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py b/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py index 01eb5227c1..f6653741ed 100644 --- a/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py +++ b/src/sqlbuild/compiler/scopes/main/list_scope_declarations.py @@ -2,7 +2,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.report_operations import list_folder +from sqlbuild.compiler.scopes._helpers.reports import list_folder from sqlbuild.compiler.scopes.models import ( ResourceIdentity, ScopeListResult, diff --git a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py index ee84e2c18d..587de54257 100644 --- a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py @@ -11,7 +11,8 @@ from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( scope_index_with_compile_usages, ) -from sqlbuild.compiler.compile.models import CompileAdapterContext, CompileProjectInputs +from sqlbuild.compiler.compile.models import CompileAdapterContext +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier @@ -36,11 +37,7 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S """Load or build scope facts without credentials, a connection, or warehouse access.""" resolved_project_dir: Path = project_dir.resolve() - fingerprint: str | None - try: - fingerprint = scope_index_fingerprint(project_dir=resolved_project_dir) - except (OSError, UnicodeError, ValueError): - fingerprint = None + fingerprint: str | None = scope_index_fingerprint(project_dir=resolved_project_dir) if not no_cache and fingerprint is not None: cached: ScopeIndex | None = read_cached_scope_index( project_dir=resolved_project_dir, fingerprint=fingerprint @@ -60,7 +57,7 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S ) adapter: BaseAdapter = resolve_adapter(adapter_name=adapter_name) sql_lexical_syntax = adapter.sql_lexical_syntax - compile_inputs: CompileProjectInputs = compile_frontier( + compile_inputs: CompileInputsView = compile_frontier( until=CompilerStage.COMPILE_PROJECT_INPUTS, python_stage=partial( build_compile_inputs, diff --git a/src/sqlbuild/compiler/scopes/main/preview_scope_move.py b/src/sqlbuild/compiler/scopes/main/preview_scope_move.py index d180eacd48..6debf20736 100644 --- a/src/sqlbuild/compiler/scopes/main/preview_scope_move.py +++ b/src/sqlbuild/compiler/scopes/main/preview_scope_move.py @@ -2,7 +2,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.report_operations import build_move_preview +from sqlbuild.compiler.scopes._helpers.reports import build_move_preview from sqlbuild.compiler.scopes.models import ( MovePreview, ResourceIdentity, diff --git a/src/sqlbuild/compiler/scopes/main/query_scope_report.py b/src/sqlbuild/compiler/scopes/main/query_scope_report.py index fce46031d2..78a620647f 100644 --- a/src/sqlbuild/compiler/scopes/main/query_scope_report.py +++ b/src/sqlbuild/compiler/scopes/main/query_scope_report.py @@ -4,7 +4,7 @@ from pathlib import PurePath -from sqlbuild.compiler.scopes._helpers.report_query import build_scope_report +from sqlbuild.compiler.scopes._helpers.reports import build_scope_report from sqlbuild.compiler.scopes.models import ( DeclarationIdentity, ResourceIdentity, diff --git a/src/sqlbuild/compiler/scopes/main/scope_metadata.py b/src/sqlbuild/compiler/scopes/main/scope_metadata.py index bee8ae11f9..6c65b50209 100644 --- a/src/sqlbuild/compiler/scopes/main/scope_metadata.py +++ b/src/sqlbuild/compiler/scopes/main/scope_metadata.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.scopes._helpers.report_projection import build_projection +from sqlbuild.compiler.scopes._helpers.report_rows import build_projection from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.compiler.scopes.types import JsonValue diff --git a/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py b/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py index 508a101e69..08ab2669dc 100644 --- a/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py +++ b/src/sqlbuild/compiler/scopes/main/serialize_scope_report.py @@ -1,6 +1,6 @@ """Public canonical scope report serialization operation.""" -from sqlbuild.compiler.scopes._helpers.report_operations import serialize_result +from sqlbuild.compiler.scopes._helpers.report_rows import serialize_result from sqlbuild.compiler.scopes.models import ScopeBrowseResult, ScopeListResult, ScopeReport diff --git a/src/sqlbuild/compiler/scopes/models.py b/src/sqlbuild/compiler/scopes/models.py index d566889487..64cbab6ccd 100644 --- a/src/sqlbuild/compiler/scopes/models.py +++ b/src/sqlbuild/compiler/scopes/models.py @@ -141,16 +141,6 @@ class NativeDeclarationValues: dependencies: tuple[DeclarationIdentity, ...] = () -@dataclass(frozen=True) -class RelationshipFact: - """Relationship names one test block or scenario authors, before grants are resolved.""" - - resource: ResourceIdentity - expected_models: tuple[str, ...] = field(default_factory=tuple) - called_macros: tuple[str, ...] = field(default_factory=tuple) - tested_macros: tuple[str, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class VisibilityRecord: """A resolved positive visibility fact with stable provenance.""" @@ -254,6 +244,7 @@ class ScopeLookup: usages_by_declaration: Mapping[DeclarationIdentity, tuple[UsageRecord, ...]] grants_by_resource: Mapping[ResourceIdentity, tuple[GrantRecord, ...]] visibility_index: DeclarationVisibilityIndex + native: object | None = field(default=None, compare=False, repr=False) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/scopes/types.py b/src/sqlbuild/compiler/scopes/types.py index 4b059a3ff4..5e18d6c16a 100644 --- a/src/sqlbuild/compiler/scopes/types.py +++ b/src/sqlbuild/compiler/scopes/types.py @@ -3,9 +3,18 @@ from __future__ import annotations from enum import StrEnum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import PurePath + + from sqlbuild.compiler.scopes.models import ( + DeclarationIdentity, + DeclarationRecord, + ResourceIdentity, + ) type JsonValue = None | bool | int | float | str | list[JsonValue] | dict[str, JsonValue] -type JsonObject = dict[str, object] type NativeIdentityRow = tuple[str, str] type NativeDeclarationIdentityRow = tuple[str, str, NativeIdentityRow | None] type NativeResourceRow = tuple[str, str, str, str] @@ -21,13 +30,87 @@ str | None, list[NativeDeclarationIdentityRow], ] - - -class ScopeCacheIdentityType(StrEnum): - """Identity discriminator values persisted by the scope cache codec.""" - - RESOURCE = "resource" - DECLARATION = "declaration" +type NativeQualifiedRow = NativeDeclarationIdentityRow +type NativeUsageKeyRow = tuple[ + NativeQualifiedRow, NativeQualifiedRow, str, NativeQualifiedRow | None, str | None +] +type NativeRootRow = tuple[str, str, str | None] +type NativeResourceRecordRow = tuple[NativeIdentityRow, str, NativeRootRow] +type ScopeDeclarationPairs[Value] = tuple[tuple[DeclarationRecord, Value], ...] +type NativeResolvedRows = tuple[ + list[tuple[int, str, int | None]], list[tuple[int, str]], str | None +] +type ScopeQueryTarget = str | PurePath | ResourceIdentity | DeclarationIdentity +type NativeScopeTargetRow = tuple[NativeQualifiedRow | None, str, bool] +type NativeScopeFiltersRow = tuple[ + list[str], bool, str | None, list[str], str | None, bool, int, str, str | None, int, int, str +] +type NativeScopeValuesRow = tuple[ + tuple[list[str], list[NativeQualifiedRow], str] | None, + tuple[list[str], str] | None, + tuple[str, str | None, int | None, bool, str | None] | None, +] +type NativeScopeCompletenessRow = tuple[bool, bool, bool, bool, bool, bool] +type NativeScopeDiagnosticRow = tuple[ + str, + str, + str, + str | None, + int | None, + int | None, + NativeQualifiedRow | None, + NativeIdentityRow | None, +] +type NativeScopeRaisedRow = tuple[int | None, tuple[str, str, str | None] | None] +type NativeScopeDeclarationRow = tuple[ + tuple[str, str, str, str | None, tuple[str, int, int], str, str | None], + tuple[tuple[str, str | None] | None, str | None, list[tuple[str, object]]], + tuple[list[str], list[str], list[str], str | None, str | None, list[str]], +] +type NativeScopeSectionRow = tuple[ + str, int, int, bool, int, bool, bool, str | None, str | None, int +] +type NativeScopeResourceReportRow = tuple[str, str | None, str | None, bool, bool, int] +type NativeScopeExplanationRow = tuple[NativeScopeDeclarationRow | None, bool] +type NativeScopeMoveRow = tuple[ + str, + str, + str | None, + list[NativeScopeDeclarationRow], + list[NativeScopeDeclarationRow], + list[NativeScopeDeclarationRow], + list[str], + list[NativeScopeDeclarationRow], + list[NativeScopeDeclarationRow], + bool, +] +type NativeScopeFolderRow = tuple[str, str, int, int, int, list[tuple[str, int]]] +type NativeScopeReportRow[Diagnostic] = tuple[ + NativeScopeResourceReportRow, + list[tuple[str, str, int]], + list[NativeScopeDeclarationRow], + list[NativeScopeDeclarationRow], + list[NativeScopeDeclarationRow], + list[NativeScopeDeclarationRow], + NativeScopeFiltersRow, + list[NativeScopeSectionRow], + NativeScopeExplanationRow | None, + NativeScopeMoveRow | None, + list[Diagnostic], + bool, +] +type NativeScopeBrowseRow[Diagnostic] = tuple[ + str, list[NativeScopeFolderRow], list[Diagnostic], bool +] +type NativeScopeListRow[Diagnostic] = tuple[ + str, list[NativeScopeDeclarationRow], NativeScopeSectionRow, list[Diagnostic] +] +type NativeScopeFailureRow = tuple[str, str] | None +type NativePlacementRows = tuple[ + list[tuple[NativeIdentityRow, str, NativeRootRow]], + list[tuple[NativeQualifiedRow, str, int, int, str, NativeRootRow, str | None]], + list[tuple[NativeQualifiedRow, NativeQualifiedRow, NativeQualifiedRow | None]], +] class ResourceKind(StrEnum): diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py index b83283daa8..f623cf26ea 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py @@ -8,12 +8,18 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs from sqlbuild.compiler.compile.constants import UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, CompiledResourceType, DiagnosticPhase, DiagnosticSeverity, ) +from sqlbuild.compiler.compiled_project.main._with_binding_diagnostics import ( + with_binding_diagnostics, +) +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.semantic_checks._helpers.payloads import diagnostic_ids, lineage_payload from sqlbuild.spec.contracts.models import SourceLocation @@ -27,11 +33,11 @@ def complete_native_diagnostics( *, - project: CompiledProject, + project: CompiledProjectView, catalog: object, session: Any | None, session_models: frozenset[str], -) -> CompiledProject: +) -> CompiledProjectView: """The recovered, explained and opt-out-checked project; a native internal failure raises.""" ids: dict[CompilerDiagnostic, int] = diagnostic_ids(project) @@ -52,11 +58,11 @@ def complete_native_diagnostics( diagnostics: list[CompilerDiagnostic] = [ _completed(original=project.diagnostics[row[0]], row=row) for row in completed ] - models: tuple[CompiledModel, ...] = project.models + models: tuple[CompiledModelView, ...] = project.models if model_bindings is not None: models = tuple( - replace( - model, binding_diagnostics=_positions(diagnostics=diagnostics, positions=positions) + with_binding_diagnostics( + model=model, diagnostics=_positions(diagnostics=diagnostics, positions=positions) ) for model, positions in zip(project.models, model_bindings, strict=True) ) @@ -71,7 +77,7 @@ def complete_native_diagnostics( model=project.models[index], message=message, note=note, help_text=help_text ) ) - return replace(project, models=models, diagnostics=tuple(final)) + return project_with(project=project, models=models, diagnostics=tuple(final)) def _positions( @@ -100,7 +106,9 @@ def _diagnostic_payload( ) -def _model_payload(*, model: CompiledModel, lineage: bool, from_session: bool) -> tuple[Any, ...]: +def _model_payload( + *, model: CompiledModelView, lineage: bool, from_session: bool +) -> tuple[Any, ...]: opt_out: SourceLocation | None = model.rejected_sql_analysis_opt_out return ( model.name, @@ -140,7 +148,7 @@ def _completed(*, original: CompilerDiagnostic, row: _Completed) -> CompilerDiag def _opt_out( - *, model: CompiledModel, message: str, note: str, help_text: str + *, model: CompiledModelView, message: str, note: str, help_text: str ) -> CompilerDiagnostic: return CompilerDiagnostic( phase=DiagnosticPhase.COMPILE, @@ -156,7 +164,7 @@ def _opt_out( def _lineage( - *, model: CompiledModel, lineage: bool, from_session: bool + *, model: CompiledModelView, lineage: bool, from_session: bool ) -> list[tuple[str, list[tuple[str, str]]]] | None: """The model's lineage rows; None asks for the session's, and only recovery reads them.""" diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py b/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py index 5cc53c77be..5c96d1a84f 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py @@ -10,16 +10,16 @@ from sqlbuild.adapter.type_system.main.normalize_type import normalize_type from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompiledModelSqlTestPayload, - CompiledProject, - CompiledSource, CompiledSqlTest, CompilerDiagnostic, ) from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, CompiledResourceType, + CompiledSourceView, DiagnosticPhase, DiagnosticSeverity, ) @@ -28,7 +28,7 @@ METADATA_CONFIG_KEYS, METADATA_FUNCTION_REFERENCE_KINDS, ) -from sqlbuild.spec.contracts.models import SourceLocation +from sqlbuild.spec.contracts.models import SchemaAuditInstance, SourceLocation type _ErrorRow = tuple[str, str, int, int] type _NamesRow = tuple[str, list[str]] @@ -36,7 +36,7 @@ def native_metadata_diagnostics( *, - project: CompiledProject, + project: CompiledProjectView, profile: ExpressionInferenceProfile, resource_sql_analysis: bool, catalog: object, @@ -54,7 +54,7 @@ def native_metadata_diagnostics( columns_by_sql: dict[str, tuple[str, ...]] = SemanticCompletionInputs.sql_test_columns( project=project, profile=profile ) - functions: dict[str, CompiledFunction] = { + functions: dict[str, CompiledFunctionView] = { function.name.casefold(): function for function in project.functions } file_texts: dict[Path, str] = {} @@ -65,6 +65,7 @@ def native_metadata_diagnostics( files: dict[Path, int] = {path: index for index, path in enumerate(file_texts)} request: Any = ( profile.sql_analysis_dialect, + profile.quoted_identifiers_ignore_case, list(profile.function_return_types.items()), [_function_payload(key=key, function=function) for key, function in functions.items()], [(name, list(columns.items())) for name, columns in shapes.items()], @@ -82,18 +83,13 @@ def native_metadata_diagnostics( for type_sql in fallback_types: _ = normalize_type(type_sql=type_sql, dialect=profile.sql_analysis_dialect) diagnostics: list[CompilerDiagnostic] = [] - for model, (function_rows, reference_rows) in zip(project.models, model_rows, strict=True): + for model, (function_rows, audit_rows, reference_rows) in zip( + project.models, model_rows, strict=True + ): if not _checked(model): continue diagnostics.extend(_model_error(model=model, row=row) for row in function_rows) - shape: dict[str, str] | None = shapes.get(model.name) - if shape is None: - continue - diagnostics.extend( - SemanticCompletionInputs.audit_diagnostics( - model=model, project=project, shape=shape, shapes=shapes, profile=profile - ) - ) + diagnostics.extend(_audit_error(model=model, row=row) for row in audit_rows) diagnostics.extend(_model_error(model=model, row=row) for row in reference_rows) diagnostics.extend( _source_error(source=project.sources[index], row=row) for index, row in source_rows @@ -111,11 +107,13 @@ def native_metadata_diagnostics( return tuple(diagnostics) -def _function_payload(*, key: str, function: CompiledFunction) -> tuple[str, list[tuple[str, str]]]: +def _function_payload( + *, key: str, function: CompiledFunctionView +) -> tuple[str, list[tuple[str, str]]]: return (key, [(argument.name, argument.type) for argument in function.arguments]) -def _checked(model: CompiledModel) -> bool: +def _checked(model: CompiledModelView) -> bool: return model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is not False and model.binding_validated @@ -123,7 +121,7 @@ def _string(value: object) -> str | None: return value if isinstance(value, str) else None -def _model_payload(model: CompiledModel) -> tuple[Any, ...]: +def _model_payload(model: CompiledModelView) -> tuple[Any, ...]: values: dict[str, object] = model.config.values references: list[_NamesRow] = [ (key, list(SemanticCompletionInputs.column_names(values.get(key)))) @@ -163,11 +161,54 @@ def _model_payload(model: CompiledModel) -> tuple[Any, ...]: upstreams, _string(values.get("cursor")), _string(values.get("cursor_type")), + _audit_payloads(model), ) +def _audit_payloads(model: CompiledModelView) -> list[tuple[object, ...]]: + if model.schema_entry is None: + return [] + attached: list[tuple[str | None, SchemaAuditInstance]] = [ + (None, audit) for audit in model.schema_entry.audits + ] + for schema_column in model.schema_entry.columns: + attached.extend((schema_column.name, audit) for audit in schema_column.audits) + payloads: list[tuple[object, ...]] = [] + for column, audit in attached: + arguments: dict[str, object] = audit.arguments + expression: object = arguments.get("expression") + values: object = arguments.get("values") + payloads.append( + ( + column, + audit.definition_name, + expression if isinstance(expression, str) else None, + str(arguments.get("to", "")), + str(arguments.get("field", "")), + [_audit_value(value) for value in values] + if isinstance(values, (list, tuple)) + else None, + ) + ) + return payloads + + +def _audit_value(value: object) -> tuple[str, str]: + """An accepted value by Python type: null, bool, number, text or other.""" + + if value is None: + return ("null", "") + if isinstance(value, bool): + return ("bool", "true" if value else "") + if isinstance(value, int | float): + return ("number", str(value)) + if isinstance(value, str): + return ("text", value) + return ("other", "") + + def _source_payload( - *, source: CompiledSource, files: dict[Path, int] + *, source: CompiledSourceView, files: dict[Path, int] ) -> tuple[str, str | None, int]: return ( source.name, @@ -189,7 +230,7 @@ def _sql_test_payload( ) -def _model_error(*, model: CompiledModel, row: _ErrorRow) -> CompilerDiagnostic: +def _model_error(*, model: CompiledModelView, row: _ErrorRow) -> CompilerDiagnostic: code, message, line, column = row return CompilerDiagnostic( phase=DiagnosticPhase.COMPILE, @@ -204,7 +245,24 @@ def _model_error(*, model: CompiledModel, row: _ErrorRow) -> CompilerDiagnostic: ) -def _source_error(*, source: CompiledSource, row: _ErrorRow) -> CompilerDiagnostic: +def _audit_error( + *, model: CompiledModelView, row: tuple[str, str, int, int, bool] +) -> CompilerDiagnostic: + code, message, line, column, is_error = row + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR if is_error else DiagnosticSeverity.WARNING, + code=code, + message=message, + resource_type=CompiledResourceType.MODEL, + resource_name=model.name, + path=model.relative_path, + line=line, + column=column, + ) + + +def _source_error(*, source: CompiledSourceView, row: _ErrorRow) -> CompilerDiagnostic: code, message, line, column = row return CompilerDiagnostic( phase=DiagnosticPhase.COMPILE, diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py index 76d8e40a60..38ba30d904 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py @@ -4,23 +4,22 @@ from typing import Any, cast +from sqlbuild.compiler.analysis_session.types import DiagnosticRow from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, - CompiledModel, - CompiledProject, CompilerDiagnostic, DynamicColumnContractProof, ) -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -def lineage_payload(model: CompiledModel) -> list[tuple[str, list[tuple[str, str]]]]: +def lineage_payload(model: CompiledModelView) -> list[tuple[str, list[tuple[str, str]]]]: """Each output column with the `(resource, column)` pairs it reads.""" return [_output_payload(output) for output in model.fast_lineage_columns or ()] -def session_fact_models(*, project: CompiledProject, session: Any | None) -> frozenset[str]: +def session_fact_models(*, project: CompiledProjectView, session: Any | None) -> frozenset[str]: """Models whose output names and lineage are the finished session's, so need no payload.""" if session is None: @@ -46,7 +45,7 @@ def _output_payload(output: CompiledLineageColumnFact) -> tuple[str, list[tuple[ ) -def diagnostic_ids(project: CompiledProject) -> dict[CompilerDiagnostic, int]: +def diagnostic_ids(project: CompiledProjectView) -> dict[CompilerDiagnostic, int]: """One id per distinct diagnostic value, as Python's value-keyed dicts and sets see them.""" ids: dict[CompilerDiagnostic, int] = {} @@ -59,18 +58,18 @@ def diagnostic_ids(project: CompiledProject) -> dict[CompilerDiagnostic, int]: def raw_binding_ids( - binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], -) -> dict[SqlBindingDiagnostic, int]: + binding_results: dict[str, list[DiagnosticRow]], +) -> dict[DiagnosticRow, int]: """One id per distinct raw binding row value.""" - ids: dict[SqlBindingDiagnostic, int] = {} + ids: dict[DiagnosticRow, int] = {} for rows in binding_results.values(): for row in rows: _ = ids.setdefault(row, len(ids)) return ids -def has_blocking_binding(project: CompiledProject) -> bool: +def has_blocking_binding(project: CompiledProjectView) -> bool: """Whether any model owns a binding error, without which type recovery changes nothing.""" for model in project.models: diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py index 3976377b0c..d0b498e3c9 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py @@ -2,35 +2,36 @@ from __future__ import annotations -from dataclasses import replace from typing import Any import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.analysis_session.types import DiagnosticRow +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.semantic_checks._helpers.completion import complete_native_diagnostics from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models from sqlbuild.compiler.semantic_checks._helpers.type_recovery import recover_native_output_types from sqlbuild.compiler.semantic_checks.constants import NATIVE_SEMANTIC_NO_CATALOG -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic def completed_semantic_project( *, - project: CompiledProject, + project: CompiledProjectView, profile: ExpressionInferenceProfile, - binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + binding_results: dict[str, list[DiagnosticRow]], resource_sql_analysis: bool, session: Any | None, -) -> CompiledProject: +) -> CompiledProjectView: """The completed project; a native internal failure raises `NativeCompilerError`.""" if project.binding_catalog is None: raise _native.NativeCompilerError(NATIVE_SEMANTIC_NO_CATALOG) catalog: object = project.binding_catalog.native session_models: frozenset[str] = session_fact_models(project=project, session=session) - recovered: CompiledProject = recover_native_output_types( + recovered: CompiledProjectView = recover_native_output_types( project=project, binding_results=binding_results, catalog=catalog, @@ -44,7 +45,7 @@ def completed_semantic_project( catalog=catalog, ) return complete_native_diagnostics( - project=replace(recovered, diagnostics=(*recovered.diagnostics, *metadata)), + project=project_with(project=recovered, diagnostics=(*recovered.diagnostics, *metadata)), catalog=catalog, session=session, session_models=session_models, diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py index 868a91e6b6..582011908b 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py @@ -6,14 +6,18 @@ from typing import Any import sqlbuild._native as _native +from sqlbuild.compiler.analysis_session.types import DiagnosticRow from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, CompilerDiagnostic, - InferredColumn, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) +from sqlbuild.compiler.compiled_project.main._with_type_recovery import with_type_recovery +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.semantic_checks._helpers.payloads import ( diagnostic_ids, has_blocking_binding, @@ -25,25 +29,25 @@ UNKNOWN_RECOVERED_TYPE, ) from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlBindingResult +from sqlbuild.compiler.sql_analysis.models import SqlBindingResult type _Revised = list[tuple[str, int | None, int | None]] def recover_native_output_types( *, - project: CompiledProject, - binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + project: CompiledProjectView, + binding_results: dict[str, list[DiagnosticRow]], catalog: object, session: Any | None, session_models: frozenset[str], -) -> CompiledProject: +) -> CompiledProjectView: """The project with recovered output types; a native internal failure raises.""" if not has_blocking_binding(project): return project ids: dict[CompilerDiagnostic, int] = diagnostic_ids(project) - raw_ids: dict[SqlBindingDiagnostic, int] = raw_binding_ids(binding_results) + raw_ids: dict[DiagnosticRow, int] = raw_binding_ids(binding_results) request: Any = ( project.sql_analysis_dialect, [ @@ -80,19 +84,19 @@ def recover_native_output_types( _with_note(diagnostic=project.diagnostics[index], note=note) for index, note in kept ) poisoned_outputs: frozenset[tuple[str, str]] = frozenset(poisoned) - models: tuple[CompiledModel, ...] = tuple( - _recovered_model( + models: tuple[CompiledModelView, ...] = tuple( + with_type_recovery( model=model, - bindings=_positions(diagnostics=diagnostics, positions=positions), + diagnostics=_positions(diagnostics=diagnostics, positions=positions), poisoned=poisoned_outputs, ) for model, positions in zip(project.models, model_bindings, strict=True) ) - return replace(project, models=models, diagnostics=diagnostics) + return project_with(project=project, models=models, diagnostics=diagnostics) def _revalidated( - *, project: CompiledProject, revalidated: list[int], poisoned: list[tuple[str, str]] + *, project: CompiledProjectView, revalidated: list[int], poisoned: list[tuple[str, str]] ) -> list[_Revised]: """Python's revalidation of each planned model with unknown types for poisoned outputs.""" @@ -116,14 +120,14 @@ def _revalidated( def _model_payload( *, - model: CompiledModel, - binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + model: CompiledModelView, + binding_results: dict[str, list[DiagnosticRow]], ids: dict[CompilerDiagnostic, int], - raw_ids: dict[SqlBindingDiagnostic, int], + raw_ids: dict[DiagnosticRow, int], from_session: bool, ) -> tuple[Any, ...]: - raws: tuple[SqlBindingDiagnostic, ...] = ( - binding_results.get(model.name, ()) if model.binding_diagnostics else () + raws: list[DiagnosticRow] = ( + binding_results.get(model.name, []) if model.binding_diagnostics else [] ) return ( model.name, @@ -134,7 +138,7 @@ def _model_payload( [reference.ref_name for reference in model.references], None if from_session else lineage_payload(model), [(ids[item], item.code, item.message, item.is_error) for item in model.binding_diagnostics], - [(raw_ids[raw], raw.code, raw.message, raw.start, raw.end) for raw in raws], + [(raw_ids[raw], raw[0], raw[1], raw[4], raw[5]) for raw in raws], ) @@ -148,27 +152,6 @@ def _positions( return tuple(diagnostics[position] for position in positions) -def _recovered_model( - *, - model: CompiledModel, - bindings: tuple[CompilerDiagnostic, ...], - poisoned: frozenset[tuple[str, str]], -) -> CompiledModel: - columns: tuple[InferredColumn, ...] | None = None - if model.inferred_columns is not None: - columns = tuple( - replace(column, type=None) if (model.name, column.name) in poisoned else column - for column in model.inferred_columns - ) - unchecked: frozenset[str] = frozenset(column for name, column in poisoned if name == model.name) - return replace( - model, - binding_diagnostics=bindings, - inferred_columns=columns, - unchecked_output_columns=unchecked, - ) - - def _unknown_shapes( *, shapes: dict[str, dict[str, str]], poisoned: list[tuple[str, str]] ) -> dict[str, dict[str, str]]: diff --git a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py index 8ebffec6a9..e224007355 100644 --- a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py +++ b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py @@ -5,19 +5,19 @@ from typing import Any from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.analysis_session.types import DiagnosticRow +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.semantic_checks._helpers.stage import completed_semantic_project -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic def complete_native_semantic_diagnostics( *, - project: CompiledProject, + project: CompiledProjectView, profile: ExpressionInferenceProfile, - binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + binding_results: dict[str, list[DiagnosticRow]], resource_sql_analysis: bool, session: Any | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """The completed project; a native internal failure raises `NativeCompilerError`.""" return completed_semantic_project( diff --git a/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py b/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py index b40a25cde0..0c895fef2d 100644 --- a/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py +++ b/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py @@ -3,7 +3,7 @@ from __future__ import annotations import sqlbuild._native as native_module -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.sql_test_glue._helpers.assembly import native_sql_test_assemblies from sqlbuild.compiler.sql_test_glue.models import ( NativeSqlTestAssembly, @@ -12,7 +12,7 @@ from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestAssemblyRow -def assemble_native_sql_tests(*, inputs: CompileProjectInputs) -> tuple[NativeSqlTestAssembly, ...]: +def assemble_native_sql_tests(*, inputs: CompileInputsView) -> tuple[NativeSqlTestAssembly, ...]: """Return one assembly per test input, with the error its assembly raises, if any.""" rows: list[NativeSqlTestAssemblyRow] = native_module.assemble_compiled_sql_tests( diff --git a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py index 0304327f4f..f43a223421 100644 --- a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py +++ b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py @@ -3,7 +3,8 @@ from __future__ import annotations from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.compile.models import CompiledSqlTest +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.main.execution._plan_compiled_sql_test_artifacts import ( plan_compiled_sql_test_artifacts, ) @@ -12,7 +13,7 @@ def plan_native_sql_test_artifacts( *, - project: CompiledProject, + project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter, sql_analysis_enabled: bool, diff --git a/src/sqlbuild/compiler/sql_test_glue/models.py b/src/sqlbuild/compiler/sql_test_glue/models.py index 69d1bdc130..94cdb880ee 100644 --- a/src/sqlbuild/compiler/sql_test_glue/models.py +++ b/src/sqlbuild/compiler/sql_test_glue/models.py @@ -5,12 +5,11 @@ from dataclasses import dataclass from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledSqlTest, - CompileModelInput, CompilerDiagnostic, CompileSqlTestInput, ) +from sqlbuild.compiler.compile.types import CompiledModelView, ModelInput from sqlbuild.compiler.sql_test_glue.types import SqlTestAssemblyFailureKind @@ -18,7 +17,7 @@ class NativeSqlTestPlanningRequest: """Compiled models and tests to plan, with the adapter facts native planning cannot derive.""" - models: tuple[CompiledModel, ...] + models: tuple[CompiledModelView, ...] rendered_model_sql: dict[int, str] functions: tuple[tuple[str, str, str, str, str], ...] tests: tuple[CompiledSqlTest, ...] @@ -36,7 +35,7 @@ class NativeSqlTestPlanningRequest: class NativeSqlTestChainRequest: """Compiled models and the tests whose unmocked model chains to resolve.""" - models: tuple[CompiledModel, ...] + models: tuple[CompiledModelView, ...] tests: tuple[CompiledSqlTest, ...] lexical_syntax: dict[str, object] @@ -45,7 +44,7 @@ class NativeSqlTestChainRequest: class NativeSqlTestAssemblyRequest: """The project's model inputs and the SQL test inputs whose compiled facts to assemble.""" - model_inputs: tuple[CompileModelInput, ...] + model_inputs: tuple[ModelInput, ...] test_inputs: tuple[CompileSqlTestInput, ...] lexical_syntax: dict[str, object] diff --git a/src/sqlbuild/executor/clone/_helpers/retention.py b/src/sqlbuild/executor/clone/_helpers/retention.py index 7597a6fec5..0740cf885c 100644 --- a/src/sqlbuild/executor/clone/_helpers/retention.py +++ b/src/sqlbuild/executor/clone/_helpers/retention.py @@ -11,7 +11,7 @@ RetentionState, ) from sqlbuild.adapter.contract.types import BuiltinAdapter, RetentionScope -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.types import RetentionPlanPhase from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.runtime.observability.classes.operation_lifecycle import ( @@ -26,7 +26,7 @@ def build_clone_retention_requests( *, - project: CompiledProject, + project: CompiledProjectView, adapter_name: str, namespace_owned: bool, selected_model_names: frozenset[str] | None = None, diff --git a/src/sqlbuild/executor/clone/main/build_destination_transient_models.py b/src/sqlbuild/executor/clone/main/build_destination_transient_models.py index 26675fb4e4..778d01aabc 100644 --- a/src/sqlbuild/executor/clone/main/build_destination_transient_models.py +++ b/src/sqlbuild/executor/clone/main/build_destination_transient_models.py @@ -1,10 +1,10 @@ """Collect destination models whose resolved table type is transient.""" -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.spec.contracts.types import TableType -def build_destination_transient_models(*, project: CompiledProject) -> frozenset[str]: +def build_destination_transient_models(*, project: CompiledProjectView) -> frozenset[str]: """Return destination model names that clones should create as transient tables.""" return frozenset( diff --git a/src/sqlbuild/executor/clone/main/build_retention_requests.py b/src/sqlbuild/executor/clone/main/build_retention_requests.py index 41da717bf8..ad05644b66 100644 --- a/src/sqlbuild/executor/clone/main/build_retention_requests.py +++ b/src/sqlbuild/executor/clone/main/build_retention_requests.py @@ -3,13 +3,13 @@ from __future__ import annotations from sqlbuild.adapter.contract.models import RetentionRequest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.executor.clone._helpers.retention import build_clone_retention_requests def build_destination_retention_requests( *, - project: CompiledProject, + project: CompiledProjectView, adapter_name: str, namespace_owned: bool, selected_model_names: frozenset[str] | None = None, diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py index 99413dc399..2fbb4127c4 100644 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ b/src/sqlbuild/executor/janitor/_helpers/classification.py @@ -9,7 +9,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.planner.main.scenarios.is_scenario_artifact_physical_name import ( is_scenario_artifact_physical_name, @@ -88,7 +88,7 @@ def collect_query_diff_artifact_candidates( def gather_janitor_warehouse_facts( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, target_schemas: set[tuple[str | None, str | None]], diff --git a/src/sqlbuild/executor/janitor/_helpers/plan.py b/src/sqlbuild/executor/janitor/_helpers/plan.py index 883c44cb42..8ed9ca6362 100644 --- a/src/sqlbuild/executor/janitor/_helpers/plan.py +++ b/src/sqlbuild/executor/janitor/_helpers/plan.py @@ -9,21 +9,21 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, - CompiledSeed, - CompiledSource, +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledSeedView, + CompiledSourceView, ) from sqlbuild.executor.janitor.constants import QUALIFIED_ORIGIN_PARTS from sqlbuild.executor.janitor.models import JanitorRelationKey -def collect_desired_keys(project: CompiledProject) -> set[JanitorRelationKey]: +def collect_desired_keys(project: CompiledProjectView) -> set[JanitorRelationKey]: """Collect relation keys that belong to the compiled project.""" keys: set[JanitorRelationKey] = set() - model: CompiledModel + model: CompiledModelView for model in project.models: keys.add( JanitorRelationKey( @@ -32,7 +32,7 @@ def collect_desired_keys(project: CompiledProject) -> set[JanitorRelationKey]: name=model.destination.name, ) ) - seed: CompiledSeed + seed: CompiledSeedView for seed in project.seeds: keys.add( JanitorRelationKey( @@ -46,7 +46,7 @@ def collect_desired_keys(project: CompiledProject) -> set[JanitorRelationKey]: def pending_migration_origins( *, - project: CompiledProject, + project: CompiledProjectView, relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]], ) -> dict[JanitorRelationKey, str]: """Return listed origins of migrations whose destination relation does not exist yet.""" @@ -57,7 +57,7 @@ def pending_migration_origins( listed.update({_folded(relation_key(item)): relation_key(item) for item in relations}) schemas: set[tuple[str | None, str | None]] = collect_target_schemas(project) pending: dict[JanitorRelationKey, str] = {} - model: CompiledModel + model: CompiledModelView for model in project.models: raw: object | None = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) destination: JanitorRelationKey = JanitorRelationKey( @@ -97,14 +97,14 @@ def _folded(key: JanitorRelationKey) -> tuple[str | None, str | None, str]: return RelationLookup.key(database=key.database, schema=key.schema, name=key.name) -def collect_target_schemas(project: CompiledProject) -> set[tuple[str | None, str | None]]: +def collect_target_schemas(project: CompiledProjectView) -> set[tuple[str | None, str | None]]: """Collect schemas where the compiled project writes relations, one spelling per schema.""" schemas: list[tuple[str | None, str | None]] = [] - model: CompiledModel + model: CompiledModelView for model in project.models: schemas.append((model.destination.database, model.destination.schema)) - seed: CompiledSeed + seed: CompiledSeedView for seed in project.seeds: schemas.append((seed.destination.database, seed.destination.schema)) return _dedupe_schema_keys(schemas) @@ -141,14 +141,14 @@ def _schema_sort_key(schema_key: tuple[str | None, str | None]) -> tuple[str, st def collect_source_schemas( *, - project: CompiledProject, + project: CompiledProjectView, default_database: str | None, default_schema: str | None, ) -> dict[tuple[str | None, str | None], set[str]]: """Collect schemas containing active configured sources.""" source_schemas: dict[tuple[str | None, str | None], set[str]] = defaultdict(set) - source: CompiledSource + source: CompiledSourceView for source in project.sources: if source.source_entry.expression is not None: continue diff --git a/src/sqlbuild/executor/janitor/main/plan.py b/src/sqlbuild/executor/janitor/main/plan.py index e681589a3a..e5e7645008 100644 --- a/src/sqlbuild/executor/janitor/main/plan.py +++ b/src/sqlbuild/executor/janitor/main/plan.py @@ -7,7 +7,7 @@ from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle from sqlbuild.executor.janitor._helpers.archive_planning import plan_janitor_archives from sqlbuild.executor.janitor._helpers.classification import ( @@ -41,7 +41,7 @@ def build_janitor_plan( *, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, connection: Any, retention_days: int, @@ -187,7 +187,7 @@ def _plan_protected_names( managed_target_schemas: set[tuple[str | None, str | None]], facts: JanitorWarehouseFacts, early_drops: tuple[str, ...], - project: CompiledProject, + project: CompiledProjectView, scope: JanitorRelationScope, now: datetime, ) -> tuple[JanitorOldNameViewPlanning, JanitorRelationScope]: @@ -212,7 +212,7 @@ def _plan_protected_names( def _protect_pending_origins( *, - project: CompiledProject, + project: CompiledProjectView, facts: JanitorWarehouseFacts, scope: JanitorRelationScope, old_names: JanitorOldNameViewPlanning, diff --git a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py index 1f29f25ef8..dcce6ff0dd 100644 --- a/src/sqlbuild/executor/old_name_views/_helpers/janitor.py +++ b/src/sqlbuild/executor/old_name_views/_helpers/janitor.py @@ -14,7 +14,7 @@ resolve_qualified_name_parts, ) from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.migrations.constants import ( MIGRATION_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, @@ -281,7 +281,7 @@ def _view(history: OldNameViewHistory) -> JanitorOldNameView: ) -def project_destinations(project: CompiledProject) -> dict[JanitorRelationKey, str]: +def project_destinations(project: CompiledProjectView) -> dict[JanitorRelationKey, str]: """Map each project model's destination to the model's name.""" return { diff --git a/src/sqlbuild/executor/old_name_views/main/_plan_old_name_view_cleanup.py b/src/sqlbuild/executor/old_name_views/main/_plan_old_name_view_cleanup.py index ba7c45882e..790cd5560b 100644 --- a/src/sqlbuild/executor/old_name_views/main/_plan_old_name_view_cleanup.py +++ b/src/sqlbuild/executor/old_name_views/main/_plan_old_name_view_cleanup.py @@ -7,7 +7,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.executor.janitor.models import JanitorOldNameViewPlanning, JanitorRelationScope from sqlbuild.executor.old_name_views._helpers.janitor import ( plan_old_name_views, @@ -24,7 +24,7 @@ def plan_old_name_view_cleanup( relations_by_schema: dict[tuple[str | None, str | None], tuple[RelationInfo, ...]], target_name: str | None, early_drops: tuple[str, ...], - project: CompiledProject, + project: CompiledProjectView, scope: JanitorRelationScope, now: datetime, ) -> tuple[JanitorOldNameViewPlanning, JanitorRelationScope]: diff --git a/src/sqlbuild/integrations/dbt/_helpers/graph/core.py b/src/sqlbuild/integrations/dbt/_helpers/graph/core.py index 15c42da1a1..5ebc65201f 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/graph/core.py +++ b/src/sqlbuild/integrations/dbt/_helpers/graph/core.py @@ -3,11 +3,13 @@ from __future__ import annotations from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledObjectKey, - CompiledProject, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.graph.main.invert_edges import invert_edges from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( @@ -24,7 +26,7 @@ def build_dbt_combined_graph( - *, manifest: DbtManifestIndex, project: CompiledProject + *, manifest: DbtManifestIndex, project: CompiledProjectView ) -> DbtCombinedGraph: """Build a downstream-only combined graph from dbt manifest and SQLBuild compile.""" @@ -133,18 +135,18 @@ def _add_sqlbuild_model_edges( *, upstream: dict[DbtCombinedGraphKey, list[DbtCombinedGraphKey]], manifest: DbtManifestIndex, - project: CompiledProject, + project: CompiledProjectView, ) -> dict[DbtCombinedGraphKey, list[DbtCombinedGraphKey]]: sqlbuild_model_names: frozenset[str] = frozenset(model.name for model in project.models) dbt_refs_by_model_name: dict[str, list[DbtManifestModel]] = {} - sqlbuild_model: CompiledModel + sqlbuild_model: CompiledModelView dbt_model: DbtManifestModel for sqlbuild_model, dbt_model in resolve_sqlbuild_model_dbt_refs( project=project, manifest=manifest, ): dbt_refs_by_model_name.setdefault(sqlbuild_model.name, []).append(dbt_model) - model: CompiledModel + model: CompiledModelView for model in project.models: key: DbtCombinedGraphKey = sqlbuild_model_graph_key(model.name) upstream.setdefault(key, []) diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py index 0e13550edd..77d0698d31 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py +++ b/src/sqlbuild/integrations/dbt/_helpers/manifest/compile_refs.py @@ -3,14 +3,10 @@ from __future__ import annotations import json -import re from pathlib import Path from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.references.main.reference_call_prefix_pattern_text import ( - reference_call_prefix_pattern_text, -) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.integrations.dbt._helpers.manifest.core import ( build_dbt_manifest_index, @@ -18,11 +14,6 @@ ) from sqlbuild.integrations.dbt.models import DbtManifestIndex -_DBT_REF_PATTERN: re.Pattern[str] = re.compile( - rf'{reference_call_prefix_pattern_text(SqlReferenceKind.DBT_REF)}\s*"([^"]+)"\s*' - r'(?:,\s*"([^"]+)"\s*)?\)' -) - def build_compile_dbt_manifest_index(*, manifest_contents: str | None) -> DbtManifestIndex | None: """Build a dbt manifest index from discovered manifest contents.""" diff --git a/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py b/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py index 4fe28bdc11..3a63266d32 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py +++ b/src/sqlbuild/integrations/dbt/_helpers/manifest/sqlbuild_refs.py @@ -4,7 +4,7 @@ from collections.abc import Sequence -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.integrations.dbt._helpers.manifest.core import resolve_dbt_manifest_model from sqlbuild.integrations.dbt.models import DbtManifestIndex, DbtManifestModel @@ -12,17 +12,17 @@ def resolve_sqlbuild_model_dbt_refs( *, - project: CompiledProject, + project: CompiledProjectView, manifest: DbtManifestIndex, selected_model_names: Sequence[str] | None = None, -) -> tuple[tuple[CompiledModel, DbtManifestModel], ...]: +) -> tuple[tuple[CompiledModelView, DbtManifestModel], ...]: """Return resolved dbt refs used directly by SQLBuild models.""" selected_names: frozenset[str] | None = ( None if selected_model_names is None else frozenset(selected_model_names) ) - refs: list[tuple[CompiledModel, DbtManifestModel]] = [] - model: CompiledModel + refs: list[tuple[CompiledModelView, DbtManifestModel]] = [] + model: CompiledModelView for model in project.models: if selected_names is not None and model.name not in selected_names: continue diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 0a2aad61ec..05f16f4124 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -9,7 +9,7 @@ from typing import TextIO from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project @@ -144,7 +144,7 @@ def compile_dbt_interop_project( adapter: BaseAdapter = resolve_dbt_interop_adapter( adapter_name=adapter_name, project_dir=project_dir ) - project: CompiledProject = build_compiled_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discovered_inputs, adapter=adapter, no_sql_validation=no_sql_validation, diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py index b081fbe88e..3153120509 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/plan_output.py @@ -11,7 +11,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands.main.execution.command_warehouse import apply_command_warehouse from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.resolve_model_deferral_inputs import ( resolve_model_deferral_inputs, @@ -57,7 +57,7 @@ def build_sqlbuild_plan_output( if not request.selected_model_names: return None - planning_project: CompiledProject = environment.project + planning_project: CompiledProjectView = environment.project connection_config: dict[str, object] = apply_command_warehouse( config=resolve_connection_config( raw_config=build_effective_connection_config( diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py index 4a4cd8b645..350a8f07e8 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py +++ b/src/sqlbuild/integrations/dbt/_helpers/planning/orchestration.py @@ -4,9 +4,7 @@ from collections.abc import Sequence -from sqlbuild.compiler.compile.models import ( - CompiledProject, -) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( resolve_sqlbuild_model_selector_names, @@ -36,7 +34,7 @@ def plan_dbt_interop_command( *, command: DbtInteropCommand | str, - project: CompiledProject, + project: CompiledProjectView, manifest: DbtManifestIndex, graph: DbtCombinedGraph, dbt_runner: DbtRunner, @@ -111,7 +109,7 @@ def plan_dbt_interop_command( ) -def _is_dbt_anchor_term(*, term: str, project: CompiledProject) -> bool: +def _is_dbt_anchor_term(*, term: str, project: CompiledProjectView) -> bool: parsed: SelectorExpansion = split_selector_expansion(term) if not parsed.downstream: return False @@ -131,7 +129,7 @@ def _dbt_failure_detail(result: DbtLsResult) -> str | None: return detail or None -def _matches_sqlbuild_direct_selector(*, term: str, project: CompiledProject) -> bool: +def _matches_sqlbuild_direct_selector(*, term: str, project: CompiledProjectView) -> bool: model_names, _translated_path = resolve_sqlbuild_model_selector_names( project=project, term=term, @@ -141,7 +139,7 @@ def _matches_sqlbuild_direct_selector(*, term: str, project: CompiledProject) -> def _build_required_dbt_selector_terms( *, - project: CompiledProject, + project: CompiledProjectView, manifest: DbtManifestIndex, selected_model_names: Sequence[str], required_unique_ids: Sequence[str], diff --git a/src/sqlbuild/integrations/dbt/_helpers/selection/core.py b/src/sqlbuild/integrations/dbt/_helpers/selection/core.py index b612a17bcb..72a7b0b765 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/selection/core.py +++ b/src/sqlbuild/integrations/dbt/_helpers/selection/core.py @@ -4,7 +4,7 @@ from collections.abc import Mapping, Sequence -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.graph.main.path_nodes import path_nodes from sqlbuild.compiler.planner.main.selection.selector_expansion import split_selector_expansion from sqlbuild.compiler.planner.main.selection.sqlbuild_model_selectors import ( @@ -31,7 +31,7 @@ def resolve_dbt_interop_sqlbuild_selection( *, - project: CompiledProject, + project: CompiledProjectView, manifest: DbtManifestIndex | None = None, graph: DbtCombinedGraph, select: Sequence[str], @@ -135,7 +135,7 @@ def resolve_dbt_interop_sqlbuild_selection( def _resolve_path_between_keys( *, - project: CompiledProject, + project: CompiledProjectView, manifest: DbtManifestIndex | None, graph: DbtCombinedGraph, parsed: SelectorExpansion, @@ -171,7 +171,7 @@ def _path_endpoint_names(*, term: str, raw_term: str) -> tuple[str, str]: def _resolve_path_endpoint_key( *, - project: CompiledProject, + project: CompiledProjectView, manifest: DbtManifestIndex | None, endpoint: str, raw_term: str, @@ -204,7 +204,7 @@ def _resolve_path_endpoint_key( def _resolve_direct_sqlbuild_keys( - *, project: CompiledProject, term: str + *, project: CompiledProjectView, term: str ) -> tuple[frozenset[DbtCombinedGraphKey], str | None]: model_names, translated_path = resolve_sqlbuild_model_selector_names( project=project, @@ -231,7 +231,7 @@ def _add_expanded_keys( def _resolve_excluded_sqlbuild_names( - *, project: CompiledProject, exclude: Sequence[str] + *, project: CompiledProjectView, exclude: Sequence[str] ) -> frozenset[str]: excluded: set[str] = set() term: str diff --git a/src/sqlbuild/integrations/dbt/main/graph/_build_combined_graph.py b/src/sqlbuild/integrations/dbt/main/graph/_build_combined_graph.py index 81a87cb272..de93d318d6 100644 --- a/src/sqlbuild/integrations/dbt/main/graph/_build_combined_graph.py +++ b/src/sqlbuild/integrations/dbt/main/graph/_build_combined_graph.py @@ -1,12 +1,12 @@ """Build the combined dbt and SQLBuild graph.""" -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph as _build from sqlbuild.integrations.dbt.models import DbtCombinedGraph, DbtManifestIndex def build_dbt_combined_graph( - *, manifest: DbtManifestIndex, project: CompiledProject + *, manifest: DbtManifestIndex, project: CompiledProjectView ) -> DbtCombinedGraph: """Build the combined dbt and SQLBuild graph.""" diff --git a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py index 39a7b35781..22d3daf843 100644 --- a/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py +++ b/src/sqlbuild/integrations/dbt/main/manifest/_resolve_sqlbuild_model_refs.py @@ -2,7 +2,7 @@ from collections.abc import Sequence -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.integrations.dbt._helpers.manifest.sqlbuild_refs import ( resolve_sqlbuild_model_dbt_refs as _resolve, ) @@ -11,10 +11,10 @@ def resolve_sqlbuild_model_dbt_refs( *, - project: CompiledProject, + project: CompiledProjectView, manifest: DbtManifestIndex, selected_model_names: Sequence[str] | None = None, -) -> tuple[tuple[CompiledModel, DbtManifestModel], ...]: +) -> tuple[tuple[CompiledModelView, DbtManifestModel], ...]: """Return dbt refs used directly by SQLBuild models.""" return _resolve(project=project, manifest=manifest, selected_model_names=selected_model_names) diff --git a/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py b/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py index 5f215fe82a..4df96a516e 100644 --- a/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py +++ b/src/sqlbuild/integrations/dbt/main/planning/_plan_interop_command.py @@ -1,6 +1,6 @@ """Plan a dbt interop command.""" -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.planning.orchestration import ( plan_dbt_interop_command as _plan, ) @@ -18,7 +18,7 @@ def plan_dbt_interop_command( *, command: DbtInteropCommand | str, - project: CompiledProject, + project: CompiledProjectView, manifest: DbtManifestIndex, graph: DbtCombinedGraph, dbt_runner: DbtRunner, diff --git a/src/sqlbuild/integrations/dbt/models.py b/src/sqlbuild/integrations/dbt/models.py index 336184a5fe..8873fed831 100644 --- a/src/sqlbuild/integrations/dbt/models.py +++ b/src/sqlbuild/integrations/dbt/models.py @@ -8,7 +8,7 @@ from typing import Any, TextIO from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.integrations.dbt._helpers.selection.selector_terms import dbt_fqn_selector_term @@ -438,7 +438,7 @@ class DbtInteropCompiledProject: adapter_name: str adapter: BaseAdapter - project: CompiledProject + project: CompiledProjectView @dataclass(frozen=True) @@ -467,7 +467,7 @@ class DbtPlanEnvironment: project_dir: Path discovered_inputs: DiscoveredProjectInputs - project: CompiledProject + project: CompiledProjectView adapter: BaseAdapter adapter_name: str diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 2bc0ecb9de..e5cd625573 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -10,8 +10,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer +from sqlbuild.compiler.compile.types import CompiledProjectView, TypedSqlValueRenderer from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.lint._helpers.native_format import with_newline_style @@ -61,7 +60,7 @@ class _FixSession: reports: list[RuleFixResult] = field(default_factory=list) blocked: dict[Path, str] = field(default_factory=dict) skipped: dict[Path, str] = field(default_factory=dict) - baseline: CompiledProject | None = None + baseline: CompiledProjectView | None = None baseline_facts: CompileFacts | None = None def run(self) -> tuple[dict[Path, str], tuple[RuleFixResult, ...], list[LintViolation]]: @@ -234,7 +233,7 @@ def _verdict(self, *, files: dict[Path, str]) -> FixVerdict | None: self.baseline_facts = compile_facts( project=self.baseline, project_dir=self.project_dir ) - compiled: CompiledProject = compile_project( + compiled: CompiledProjectView = compile_project( discovered_inputs=_updated_inputs(inputs=self.inputs, files=files), adapter=self.adapter, ) diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py index 3cdf968b2a..6eccef6f92 100644 --- a/src/sqlbuild/lint/_helpers/native.py +++ b/src/sqlbuild/lint/_helpers/native.py @@ -4,6 +4,7 @@ import re import textwrap +from bisect import bisect_right from dataclasses import dataclass from pathlib import Path @@ -13,8 +14,6 @@ from sqlbuild.compiler.discovery._helpers.sql.model_files import ( parse_header_values, # noqa: FFL102 - the compiler owns the canonical native header grammar prepare_model_header_tokens, - source_line_starts, - source_position, ) from sqlbuild.compiler.discovery.exceptions import DiscoveryError, ModelSqlParseError from sqlbuild.lint.constants import ( @@ -848,3 +847,21 @@ def drop_resolved_description_faults( if violation.code != RULE_DESCRIPTION_PRESENT or violation.file_path.resolve() not in described ] + + +def source_line_starts(contents: str) -> tuple[int, ...]: + """Return the offset where each line of contents starts.""" + + starts: list[int] = [0] + index: int = contents.find("\n") + while index != -1: + starts.append(index + 1) + index = contents.find("\n", index + 1) + return tuple(starts) + + +def source_position(*, offset: int, line_starts: tuple[int, ...]) -> tuple[int, int]: + """Map a zero-based offset to a one-based line and column.""" + + line_index: int = bisect_right(line_starts, offset) - 1 + return line_index + 1, offset - line_starts[line_index] + 1 diff --git a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py index 6c2f75db11..eb7b02dc82 100644 --- a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py +++ b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py @@ -8,8 +8,6 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import ( MACRO_TOKEN, - SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN, - SQL_ARGUMENT_RAW_PARAMETER_PATTERN, SQL_INTERPOLATION_TOKEN, ) from sqlbuild.compiler.compile.models import ExpansionSpan @@ -23,6 +21,8 @@ OPENING_PAREN_CHARACTER, SENTINEL_TEMPLATE, SINGLE_QUOTE_CHARACTER, + SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN, + SQL_ARGUMENT_RAW_PARAMETER_PATTERN, SQL_ESCAPE_CHARACTER, SQL_QUOTE_CHARACTERS, TEMPLATE_INTERPOLATION_END, diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index 7ee0d132fe..947f1197ff 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -2,6 +2,8 @@ from __future__ import annotations +import re + from sqlbuild.lint.types import LintSeverity LINT_ENGINE_SQLBUILD: str = "sqlbuild" @@ -118,20 +120,7 @@ LINE_FEED: str = "\n" CARRIAGE_RETURN_LINE_FEED: str = "\r\n" TAB_CHARACTER: str = "\t" -UNIQUE_AUDIT_NAME: str = "unique" -UNIQUE_AUDIT_COLUMN_ARGUMENT: str = "column" -UNIQUE_KEY_CONFIG: str = "unique_key" -NOT_NULL_AUDIT_NAME: str = "not_null" -MATERIALIZED_CONFIG: str = "materialized" -INCREMENTAL_MATERIALIZATION: str = "incremental" -INCREMENTAL_MODE_CONFIG: str = "incremental_mode" -MICROBATCH_MODE: str = "microbatch" -INCREMENTAL_STRATEGY_CONFIG: str = "incremental_strategy" -CURSOR_CONFIG: str = "cursor" -KEYED_DELETE_INSERT_STRATEGY: str = "delete_insert" -KEY_PRESERVING_STRATEGIES: frozenset[str] = frozenset({"merge", KEYED_DELETE_INSERT_STRATEGY}) RELATION_IDENTITY_TEMPLATE: str = "{kind}:{name}" -RELATION_KIND_REF: str = "ref" RELATION_KIND_SOURCE: str = "source" RELATION_KIND_SEED: str = "seed" @@ -140,3 +129,10 @@ FIX_DEPENDENCIES_CHANGED: str = "The fix would change model dependencies" FIX_LINEAGE_CHANGED: str = "The fix would change column lineage" FIX_NEW_DIAGNOSTICS: str = "The fix would add compile diagnostics: " + +SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN: re.Pattern[str] = re.compile( + r"(?[A-Za-z_][A-Za-z0-9_]*)'" +) +SQL_ARGUMENT_RAW_PARAMETER_PATTERN: re.Pattern[str] = re.compile( + r"(?[A-Za-z_][A-Za-z0-9_]*)(?![A-Za-z0-9_])(?!\s*\()" +) diff --git a/src/sqlbuild/lint/main/compile_facts.py b/src/sqlbuild/lint/main/compile_facts.py index 43a00b0f76..43e5939eb3 100644 --- a/src/sqlbuild/lint/main/compile_facts.py +++ b/src/sqlbuild/lint/main/compile_facts.py @@ -5,13 +5,13 @@ from collections import Counter from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.lint.models import CompileFacts from sqlbuild.lint.types import DiagnosticIdentity -def compile_facts(*, project: CompiledProject, project_dir: Path) -> CompileFacts: +def compile_facts(*, project: CompiledProjectView, project_dir: Path) -> CompileFacts: """Index a compilation so two compilations can be compared file by file.""" model_paths: dict[CompiledObjectKey, Path] = { diff --git a/src/sqlbuild/lint/main/compiled_relation_keys.py b/src/sqlbuild/lint/main/compiled_relation_keys.py index 6204dc97b4..de49299728 100644 --- a/src/sqlbuild/lint/main/compiled_relation_keys.py +++ b/src/sqlbuild/lint/main/compiled_relation_keys.py @@ -2,25 +2,16 @@ from __future__ import annotations -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.lint.constants import RELATION_KIND_REF, RELATION_KIND_SEED, RELATION_KIND_SOURCE +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.lint.main.declared_relation_keys import declared_relation_keys from sqlbuild.lint.types import RelationKeys -def compiled_relation_keys(project: CompiledProject) -> RelationKeys: +def compiled_relation_keys(project: CompiledProjectView) -> RelationKeys: """Collect declared keys of compiled models, seeds and sources.""" return declared_relation_keys( - relations=( - *( - (RELATION_KIND_REF, model.name, model.schema_entry, model.config.values) - for model in project.models - ), - *((RELATION_KIND_SEED, seed.name, seed.schema_entry, {}) for seed in project.seeds), - *( - (RELATION_KIND_SOURCE, source.name, source.source_entry, {}) - for source in project.sources - ), - ) + model_rows=(model.relation_key_row for model in project.models), + seeds=(seed.schema_entry for seed in project.seeds), + sources=(source.source_entry for source in project.sources), ) diff --git a/src/sqlbuild/lint/main/declared_relation_keys.py b/src/sqlbuild/lint/main/declared_relation_keys.py index f340aa41e8..37237157d4 100644 --- a/src/sqlbuild/lint/main/declared_relation_keys.py +++ b/src/sqlbuild/lint/main/declared_relation_keys.py @@ -2,81 +2,31 @@ from __future__ import annotations -from collections.abc import Iterable, Mapping +from collections.abc import Iterable -from sqlbuild.lint.constants import ( - CURSOR_CONFIG, - INCREMENTAL_MATERIALIZATION, - INCREMENTAL_MODE_CONFIG, - INCREMENTAL_STRATEGY_CONFIG, - KEY_PRESERVING_STRATEGIES, - KEYED_DELETE_INSERT_STRATEGY, - MATERIALIZED_CONFIG, - MICROBATCH_MODE, - NOT_NULL_AUDIT_NAME, - RELATION_IDENTITY_TEMPLATE, - RELATION_KIND_REF, - RELATION_KIND_SEED, - UNIQUE_AUDIT_COLUMN_ARGUMENT, - UNIQUE_AUDIT_NAME, - UNIQUE_KEY_CONFIG, -) -from sqlbuild.lint.types import RelationKeys +import sqlbuild._native as _native +from sqlbuild.lint.constants import RELATION_KIND_SEED, RELATION_KIND_SOURCE +from sqlbuild.lint.types import RelationKeyRow, RelationKeys +from sqlbuild.spec.contracts.models import SchemaSeedEntry, SourceEntry def declared_relation_keys( - *, relations: Iterable[tuple[str, str, object, Mapping[str, object]]] + *, + model_rows: Iterable[RelationKeyRow], + seeds: Iterable[SchemaSeedEntry], + sources: Iterable[SourceEntry], ) -> RelationKeys: - """Collect keys that prove one row per key from `(kind, name, entry, config)` relations.""" - - keys: dict[str, list[tuple[str, ...]]] = {} - for kind, name, entry, values in relations: - found: list[tuple[str, ...]] = _trusted_unique_key(values=values) - found.extend((column,) for column in _non_null_unique_columns(entry=entry)) - if not found: - continue - kinds: tuple[str, ...] = ( - (kind, RELATION_KIND_REF) if kind == RELATION_KIND_SEED else (kind,) - ) - for identity_kind in kinds: - identity: str = RELATION_IDENTITY_TEMPLATE.format(kind=identity_kind, name=name) - keys.setdefault(identity, []).extend(found) - return {name: tuple(dict.fromkeys(found)) for name, found in sorted(keys.items())} - - -def _non_null_unique_columns(*, entry: object) -> list[str]: - """Columns with a `unique` audit that are also non-null by audit or `nullable false`.""" - - audited: dict[str, set[str]] = {UNIQUE_AUDIT_NAME: set(), NOT_NULL_AUDIT_NAME: set()} - unique: list[str] = [] - for column in getattr(entry, "columns", None) or (): - names: set[str] = {audit.definition_name for audit in getattr(column, "audits", None) or ()} - if UNIQUE_AUDIT_NAME in names: - unique.append(str(column.name)) - if NOT_NULL_AUDIT_NAME in names or getattr(column, "nullable", None) is False: - audited[NOT_NULL_AUDIT_NAME].add(str(column.name).casefold()) - for audit in getattr(entry, "audits", None) or (): - argument: object = audit.arguments.get(UNIQUE_AUDIT_COLUMN_ARGUMENT) - if audit.definition_name in audited and isinstance(argument, str): - audited[audit.definition_name].add(argument.casefold()) - if audit.definition_name == UNIQUE_AUDIT_NAME: - unique.append(argument) - return [column for column in unique if column.casefold() in audited[NOT_NULL_AUDIT_NAME]] - - -def _trusted_unique_key(*, values: Mapping[str, object]) -> list[tuple[str, ...]]: - """`unique_key` only where the incremental strategy keeps one row per key.""" - - strategy: str = str(values.get(INCREMENTAL_STRATEGY_CONFIG, "")).lower() - trusted: bool = ( - str(values.get(MATERIALIZED_CONFIG, "")).lower() == INCREMENTAL_MATERIALIZATION - and str(values.get(INCREMENTAL_MODE_CONFIG, "")).lower() != MICROBATCH_MODE - and strategy in KEY_PRESERVING_STRATEGIES - and not (strategy == KEYED_DELETE_INSERT_STRATEGY and values.get(CURSOR_CONFIG)) - ) - value: object = values.get(UNIQUE_KEY_CONFIG) if trusted else None - if isinstance(value, str): - return [(value,)] - if isinstance(value, list | tuple) and value and all(isinstance(item, str) for item in value): - return [tuple(str(item) for item in value)] - return [] + """Keys that prove one row per key, from model key rows and seed and source entries.""" + + rows: list[RelationKeyRow] = [ + *model_rows, + *(_native.relation_key_row(RELATION_KIND_SEED, seed.name, {}, seed) for seed in seeds), + *( + _native.relation_key_row(RELATION_KIND_SOURCE, source.name, {}, source) + for source in sources + ), + ] + keys: RelationKeys = {} + for identity, found in _native.declared_relation_keys(rows).items(): + keys[identity] = tuple(map(tuple, found)) + return keys diff --git a/src/sqlbuild/lint/main/fix_verdict.py b/src/sqlbuild/lint/main/fix_verdict.py index dc7d12aaec..a68ccfc252 100644 --- a/src/sqlbuild/lint/main/fix_verdict.py +++ b/src/sqlbuild/lint/main/fix_verdict.py @@ -5,7 +5,7 @@ from collections import Counter from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.lint.constants import ( FIX_COLUMNS_CHANGED, FIX_COLUMNS_UNKNOWN, @@ -52,7 +52,7 @@ def fix_verdict( def _model_change( - *, before: CompiledModel, after: CompiledModel, require_columns: bool + *, before: CompiledModelView, after: CompiledModelView, require_columns: bool ) -> str | None: """Edited models need known, equal columns; others must not change; absent lineage matches.""" diff --git a/src/sqlbuild/lint/main/quoted_value_end.py b/src/sqlbuild/lint/main/quoted_value_end.py deleted file mode 100644 index f397b2de2a..0000000000 --- a/src/sqlbuild/lint/main/quoted_value_end.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public end-offset finder for quoted MODEL header values.""" - -from __future__ import annotations - -from sqlbuild.lint._helpers.native import quoted_value_end as _quoted_value_end - - -def quoted_value_end(*, text: str, start: int) -> int: - """Return the offset just past the quoted value that opens at start.""" - - return _quoted_value_end(text=text, start=start) diff --git a/src/sqlbuild/lint/models.py b/src/sqlbuild/lint/models.py index 30539080cd..c036768590 100644 --- a/src/sqlbuild/lint/models.py +++ b/src/sqlbuild/lint/models.py @@ -7,7 +7,8 @@ from dataclasses import dataclass, field from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey, ExpansionSpan +from sqlbuild.compiler.compile.models import CompiledObjectKey, ExpansionSpan +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint.constants import ( DEFAULT_MAX_LITERAL_LENGTH, @@ -220,7 +221,7 @@ def warnings(self) -> tuple[LintViolation, ...]: class CompileFacts: """Per-model outputs and per-owner diagnostics of one compilation.""" - models: dict[CompiledObjectKey, CompiledModel] + models: dict[CompiledObjectKey, CompiledModelView] model_paths: dict[CompiledObjectKey, Path] diagnostics: dict[Path, Counter[DiagnosticIdentity]] diff --git a/src/sqlbuild/lint/types.py b/src/sqlbuild/lint/types.py index 82135aafcf..1255e5989f 100644 --- a/src/sqlbuild/lint/types.py +++ b/src/sqlbuild/lint/types.py @@ -20,6 +20,18 @@ class LintSeverity(StrEnum): type RelationKeys = dict[str, tuple[tuple[str, ...], ...]] +type RelationKeyRow = tuple[ + str, + str, + str, + str, + str, + bool, + list[str] | None, + list[tuple[str, list[str], bool]], + list[tuple[str, str | None]], +] + type DiagnosticIdentity = tuple[str, str, str, str] type NativeLintPreparationRequest = LintPreparationRequest diff --git a/src/sqlbuild/rule_engine/_helpers/engine/config.py b/src/sqlbuild/rule_engine/_helpers/engine/config.py index d8cbe73b35..e69c2e8a3e 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/config.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/config.py @@ -6,7 +6,8 @@ from pathlib import Path from typing import cast -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME from sqlbuild.compiler.planner.main.selection.scope import build_planner_scope from sqlbuild.compiler.planner.models import PlannerScope @@ -135,7 +136,9 @@ def _reject_forbidden_exceptions(config: RulesConfig) -> None: ) -def resolve_rule_ignore_selectors(*, config: RulesConfig, project: CompiledProject) -> RulesConfig: +def resolve_rule_ignore_selectors( + *, config: RulesConfig, project: CompiledProjectView +) -> RulesConfig: """Resolve configured resource selectors into deterministic finding paths.""" if not any(ignore.selectors for ignore in config.rule_ignores): return config @@ -162,7 +165,7 @@ def resolve_rule_ignore_selectors(*, config: RulesConfig, project: CompiledProje return replace(config, rule_ignores=tuple(resolved)) -def _resource_paths(*, project: CompiledProject) -> dict[CompiledObjectKey, Path]: +def _resource_paths(*, project: CompiledProjectView) -> dict[CompiledObjectKey, Path]: paths: dict[CompiledObjectKey, Path] = { model.key: model.relative_path for model in project.models } diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index ccd1fe39d2..2299101a90 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -14,6 +14,8 @@ import orjson from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( @@ -69,7 +71,7 @@ class _RuleCache: def evaluate_custom_rules_cached( *, - project: CompiledProject, + project: CompiledProjectView, config: RulesConfig, project_dir: Path, rules: tuple[Rule, ...], @@ -84,7 +86,7 @@ def evaluate_custom_rules_cached( custom: tuple[Rule, ...] = tuple(sorted(rules, key=lambda rule: rule.code)) if not custom: return CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) - facts_project: CompiledProject = host_project(project) + facts_project: CompiledProjectView = host_project(project) if not config.cache.enabled: uncached: CustomHostRun = run_custom_hosts( project=facts_project, @@ -192,7 +194,7 @@ def evaluate_custom_rules_cached( def _views_factory( - *, project: CompiledProject, project_dir: Path, dialect: str + *, project: CompiledProjectView, project_dir: Path, dialect: str ) -> Callable[[], RuleFactViews]: def build() -> RuleFactViews: return build_rule_fact_views(project=project, project_dir=project_dir, dialect=dialect) @@ -201,7 +203,7 @@ def build() -> RuleFactViews: def _subjects( - *, project: CompiledProject, rules: tuple[Rule, ...] + *, project: CompiledProjectView, rules: tuple[Rule, ...] ) -> dict[str, tuple[_Subject, ...]]: project_subject: tuple[_Subject, ...] = ( _Subject( @@ -209,12 +211,18 @@ def _subjects( identity=_digest([len(project.models), project.effective_target_name]), ), ) - model_subjects: tuple[_Subject, ...] = tuple( - _Subject( - key=model.relative_path.as_posix(), - identity=_digest([model.name, model.relative_path.as_posix(), _materialization(model)]), + model_subjects: tuple[_Subject, ...] = ( + tuple( + _Subject( + key=model.relative_path.as_posix(), + identity=_digest( + [model.name, model.relative_path.as_posix(), _materialization(model)] + ), + ) + for model in sorted(project.models, key=lambda item: item.relative_path.as_posix()) ) - for model in sorted(project.models, key=lambda item: item.relative_path.as_posix()) + if any(rule.subject is not RuleSubject.PROJECT for rule in rules) + else () ) return { rule.code: project_subject if rule.subject is RuleSubject.PROJECT else model_subjects @@ -222,7 +230,7 @@ def _subjects( } -def _model_paths(project: CompiledProject) -> tuple[str, ...]: +def _model_paths(project: CompiledProjectView) -> tuple[str, ...]: return tuple(sorted(model.relative_path.as_posix() for model in project.models)) @@ -274,11 +282,11 @@ def _readset_digest( return memo[index] -def host_project(project: CompiledProject) -> CompiledProject: +def host_project(project: CompiledProjectView) -> CompiledProject: """Drop project state no Rule fact reads, before digests and host payloads use it.""" return replace( - project, + compiled_project_record(project, include_sql_expansions=False), binding_catalog=None, sql_expansions={}, loaded_macros={}, @@ -296,7 +304,7 @@ def host_project(project: CompiledProject) -> CompiledProject: def _validate_paths( *, - project: CompiledProject, + project: CompiledProjectView, rules: tuple[Rule, ...], evaluated: dict[tuple[str, str], CustomHostEvaluation], ) -> None: diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index a28072f54c..7c8ae7d235 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -4,46 +4,27 @@ import inspect import json -from collections.abc import Iterator +from collections.abc import Callable from concurrent.futures import Executor, Future, ThreadPoolExecutor -from dataclasses import asdict, dataclass, replace -from decimal import Decimal +from dataclasses import asdict, replace +from functools import partial from pathlib import Path -from typing import Any, cast - -import orjson import sqlbuild._native as _native -from sqlbuild.adapter.type_system.main.types_equal import types_equal -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledModelSqlTestPayload, - CompiledProject, - CompiledSqlScenario, - CompiledSqlTest, - CompileSqlTestCte, - DynamicColumnContractProof, - InferredColumn, -) -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( custom_rule_implementation_fingerprint, custom_rule_import_closure, custom_rule_test_evidence, ) from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path -from sqlbuild.rule_engine._helpers.run.native_memo import ( - native_payload_digests, - native_request_identity, - read_native_response, - write_native_response, +from sqlbuild.rule_engine._helpers.run.native_rows import ( + build_rules_request, + evaluate_rules_request, + finalize_rules_rows, ) from sqlbuild.rule_engine.constants import ( - NATIVE_RULES_CACHE_FILE, RULES_NATIVE_API_VERSION, TYPE_PROOF_RULE_CODES, ) @@ -51,31 +32,16 @@ from sqlbuild.rule_engine.models import ( CustomRulesOutcome, Finding, + NativeRulesEvaluation, Rule, RulesConfig, RulesResult, ) -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind - - -@dataclass(frozen=True) -class _EncodedNativeRequest: - request_json: bytes - model_jsons: list[bytes] - model_digests: list[str] - - -@dataclass(frozen=True) -class _PreparedNativeRequest: - identity: str | None - reused: str | None - parsed: _native.ParsedRulesRequest | None def evaluate_native( *, - project: CompiledProject, + project: CompiledProjectView, config: RulesConfig, project_dir: Path, catalogue: tuple[Rule, ...], @@ -93,24 +59,19 @@ def evaluate_native( if custom_payloads is None else custom_payloads ) - prepared: _PreparedNativeRequest = _prepare_native_request( - encoded=_encode_native_request( - project=project, - config=config, - project_dir=project_dir, - dialect=dialect, - initial_findings=initial_findings, - custom_payloads=payloads, - ), + return _evaluate_native_rows( + project=project, + config=config, project_dir=project_dir, - cache_enabled=config.cache.enabled, - ) - with ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as pool: - custom_future: Future[CustomRulesOutcome] | None = ( - custom_outcome + dialect=dialect, + initial_findings=initial_findings, + defer_suppressions=defer_suppressions, + custom_payloads=payloads, + start_custom=( + (lambda **_: custom_outcome) if custom_payloads is not None - else start_custom_rules( - executor=pool, + else partial( + start_custom_rules, project=project, config=config, project_dir=project_dir, @@ -119,126 +80,68 @@ def evaluate_native( dialect=dialect, verify_determinism=verify_determinism, ) - ) - reused: bool = prepared.reused is not None - try: - response: object = orjson.loads( - _evaluate_request(prepared=prepared, project_dir=project_dir) - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - del prepared + ), + ) + + +def _evaluate_native_rows( + *, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + dialect: str, + initial_findings: tuple[Finding, ...], + defer_suppressions: bool, + custom_payloads: list[dict[str, object]], + start_custom: Callable[..., Future[CustomRulesOutcome] | None], +) -> RulesResult: + request: _native.NativeRulesRequest = build_rules_request( + project=project, + config=config, + project_dir=project_dir, + dialect=dialect, + initial_findings=initial_findings, + custom_payloads=custom_payloads, + include_type_proof=any( + _rule_selected(config=config, code=code) for code in TYPE_PROOF_RULE_CODES + ), + ) + with ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-custom-rules") as pool: + custom_future: Future[CustomRulesOutcome] | None = start_custom(executor=pool) + native: NativeRulesEvaluation = evaluate_rules_request(request) + del request custom: CustomRulesOutcome = ( CustomRulesOutcome(findings=(), cache_hits=0, cache_misses=0, custom_ms=0) if custom_future is None else custom_future.result() ) - if not isinstance(response, dict): - raise RulesError("native rules engine returned an invalid response") - payload: dict[str, Any] = response - if payload.get("version") != RULES_NATIVE_API_VERSION: - raise RulesError("native rules engine returned an unsupported response version") - raw_findings: object = payload.get("faults") - selected_codes: object = payload.get("selected_codes") - if not isinstance(raw_findings, list) or not isinstance(selected_codes, list): - raise RulesError("native rules engine returned invalid findings") - findings: tuple[Finding, ...] = tuple( - decode_rule_finding(value) for value in (*raw_findings, *custom.findings) + findings: tuple[Finding, ...] = ( + *native.findings, + *(decode_rule_finding(value) for value in custom.findings), ) if not defer_suppressions: findings = finalize_native_findings( config=config, project_dir=project_dir, - evaluated_codes=tuple(str(code) for code in selected_codes), + evaluated_codes=native.selected_codes, findings=findings, ) - native_hits: int = int(payload.get("cache_hits", 0)) - native_misses: int = int(payload.get("cache_misses", 0)) return RulesResult( findings=findings, - evaluated_models=int(payload.get("evaluated_models", 0)), - cache_hits=(native_hits + native_misses if reused else native_hits) + custom.cache_hits, - cache_misses=(0 if reused else native_misses) + custom.cache_misses, - built_in_ms=0 if reused else int(payload.get("built_in_ms", 0)), + evaluated_models=native.evaluated_models, + cache_hits=(native.cache_hits + native.cache_misses if native.reused else native.cache_hits) + + custom.cache_hits, + cache_misses=(0 if native.reused else native.cache_misses) + custom.cache_misses, + built_in_ms=0 if native.reused else native.built_in_ms, custom_ms=custom.custom_ms, custom_cpu_ms=custom.custom_cpu_ms, ) -def _encode_native_request( - *, - project: CompiledProject, - config: RulesConfig, - project_dir: Path, - dialect: str, - initial_findings: tuple[Finding, ...], - custom_payloads: list[dict[str, object]], -) -> _EncodedNativeRequest: - """Encode each model separately so its digest keys cached findings without a second pass.""" - - include_type_proof: bool = any( - _rule_selected(config=config, code=code) for code in TYPE_PROOF_RULE_CODES - ) - model_jsons: list[bytes] = [ - _serialise_native_request(payload) - for payload in _model_payloads( - project=project, dialect=dialect, include_type_proof=include_type_proof - ) - ] - return _EncodedNativeRequest( - request_json=_serialise_native_request( - _native_request( - project=project, - config=config, - project_dir=project_dir, - dialect=dialect, - initial_findings=initial_findings, - custom_payloads=custom_payloads, - ) - ), - model_jsons=model_jsons, - model_digests=native_payload_digests(model_jsons), - ) - - -def _native_request( - *, - project: CompiledProject, - config: RulesConfig, - project_dir: Path, - dialect: str, - initial_findings: tuple[Finding, ...], - custom_payloads: list[dict[str, object]], -) -> dict[str, object]: - return { - "version": RULES_NATIVE_API_VERSION, - "project_dir": str(project_dir.resolve()), - "dialect": dialect, - "config": _config_payload(config), - "sql_tests": _sql_test_payloads(project), - "sql_scenarios": _sql_scenario_payloads(project), - "public_enums": [ - _enum_payload(declaration) for declaration in project.public_enums.values() - ], - "public_constants": [ - _constant_payload(declaration) for declaration in project.public_constants.values() - ], - "scope_index": scope_metadata_projection(index=project.scope_index), - "initial_findings": [_finding_payload(finding) for finding in initial_findings], - "defer_suppressions": True, - "custom_rules": custom_payloads, - "rules_cache_path": str( - rules_bulk_cache_path( - project_dir=project_dir.resolve(), file_name=NATIVE_RULES_CACHE_FILE - ) - ), - } - - def start_custom_rules( *, executor: Executor, - project: CompiledProject, + project: CompiledProjectView, config: RulesConfig, project_dir: Path, catalogue: tuple[Rule, ...], @@ -284,59 +187,6 @@ def _selected_custom_rules( return tuple(rule for rule in catalogue if rule.custom and rule.code in selected) -def _serialise_native_request(request: dict[str, object]) -> bytes: - """Encode one native request part, reporting values the encoder rejects as rules errors.""" - - try: - return orjson.dumps(request, option=orjson.OPT_SORT_KEYS, default=str) - except orjson.JSONEncodeError as error: - raise RulesError(str(error)) from error - - -def _prepare_native_request( - *, encoded: _EncodedNativeRequest, project_dir: Path, cache_enabled: bool -) -> _PreparedNativeRequest: - """Resolve the memo, else decode natively so the encoded payloads die when this returns.""" - - identity: str | None = ( - native_request_identity( - request_json=encoded.request_json, model_digests=encoded.model_digests - ) - if cache_enabled - else None - ) - reused: str | None = ( - read_native_response(project_dir=project_dir, identity=identity) - if identity is not None - else None - ) - return _PreparedNativeRequest( - identity=identity, - reused=reused, - parsed=_parse_native_request(encoded) if reused is None else None, - ) - - -def _parse_native_request(encoded: _EncodedNativeRequest) -> _native.ParsedRulesRequest: - try: - return _native.parse_rules_parts( - encoded.request_json, encoded.model_jsons, encoded.model_digests - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - - -def _evaluate_request(*, prepared: _PreparedNativeRequest, project_dir: Path) -> str: - if prepared.reused is not None: - return prepared.reused - response: str = _native.evaluate_parsed_rules(cast(_native.ParsedRulesRequest, prepared.parsed)) - if prepared.identity is not None: - write_native_response( - project_dir=project_dir, identity=prepared.identity, response=response - ) - return response - - def finalize_native_findings( *, config: RulesConfig, @@ -346,25 +196,23 @@ def finalize_native_findings( ) -> tuple[Finding, ...]: """Apply exception policy once to completed SQL, native, and custom findings.""" - request: dict[str, object] = { - "version": RULES_NATIVE_API_VERSION, - "project_dir": str(project_dir.resolve()), - "config": _config_payload(config), - "evaluated_codes": evaluated_codes, - "findings": [_finding_payload(finding) for finding in findings], - } - try: - payload: object = orjson.loads( - _native.finalize_rule_findings_json(orjson.dumps(request).decode()) - ) - except (ValueError, TypeError) as error: - raise RulesError(str(error)) from error - if not isinstance(payload, list): - raise RulesError("native rules engine returned invalid finalized findings") + return _with_fixable( + original=findings, + finalized=finalize_rules_rows( + config=config, + project_dir=project_dir, + evaluated_codes=evaluated_codes, + findings=findings, + ), + ) + + +def _with_fixable( + *, original: tuple[Finding, ...], finalized: tuple[Finding, ...] +) -> tuple[Finding, ...]: fixable: frozenset[tuple[str, str, int, int, str]] = frozenset( - _finding_identity(finding) for finding in findings if finding.fixable + _finding_identity(finding) for finding in original if finding.fixable ) - finalized: tuple[Finding, ...] = tuple(decode_rule_finding(value) for value in payload) return tuple( replace(finding, fixable=True) if _finding_identity(finding) in fixable else finding for finding in finalized @@ -474,263 +322,6 @@ def _rule_selected(*, config: RulesConfig, code: str) -> bool: return selected and not ignored -def _model_payloads( - *, project: CompiledProject, dialect: str, include_type_proof: bool -) -> Iterator[dict[str, object]]: - audit_counts: dict[str, int] = {} - for audit in project.audits: - if audit.attached_target_name is not None: - audit_counts[audit.attached_target_name] = ( - audit_counts.get(audit.attached_target_name, 0) + 1 - ) - test_counts: dict[str, int] = {} - for test in project.sql_tests: - if test.mode is not SqlTestMode.MODEL: - continue - for name in test.target_model_names: - test_counts[name] = test_counts.get(name, 0) + 1 - return ( - dict( - _model_payload( - model=model, - compiled_audit_count=audit_counts.get(model.name, 0), - targeting_test_count=test_counts.get(model.name, 0), - dialect=dialect, - include_type_proof=include_type_proof, - ), - sql_analysis_disabled=not project.settings.sql_analysis - or model.config.values.get("sql_analysis") is False, - ) - for model in project.models - ) - - -def _sql_test_payloads(project: CompiledProject) -> list[dict[str, object]]: - return [_sql_test_payload(test) for test in project.sql_tests] - - -def _sql_test_payload(test: CompiledSqlTest) -> dict[str, object]: - payload: dict[str, object] = { - "source_path": (test.source_path or test.test_file.relative_path).as_posix(), - "ownership_root": (test.ownership_root or test.test_file.ownership_root).as_posix(), - "block_index": test.block_index or test.test_block.test_index, - "name": test.name, - "explicit_name": test.explicit_name, - "mode": test.mode.value, - "expected_model_names": list(test.expected_model_names), - "assertion_names": list(test.assertion_names), - "assertion_target_model_names": list(test.assertion_target_model_names), - "target_model_names": list(test.target_model_names), - "tested_resources": [ - {"kind": resource.kind.value, "name": resource.name} - for resource in test.tested_resources - ], - } - if isinstance(test.payload, CompiledModelSqlTestPayload): - payload.update( - { - "authored_ctes": _sql_test_cte_payloads(test.payload.authored_ctes), - "expected_ctes": _sql_test_cte_payloads(test.payload.expected_ctes), - "assertion_ctes": _sql_test_cte_payloads(test.payload.assertion_ctes), - "has_macro_mocks": bool(test.payload.macro_mocks), - "has_model_query_overrides": bool(test.payload.model_query_overrides), - } - ) - if test.case_name is not None: - parameter_types: dict[str, str] = { - parameter.name: parameter.value_type.value for parameter in test.parameter_schema - } - payload.update( - { - "parent_name": test.parent_name, - "case_name": test.case_name, - "case_index": test.case_index, - "case_fingerprint": test.case_fingerprint, - "parameter_schema": [ - { - "name": parameter.name, - "type": parameter.value_type.value, - "nullable": parameter.nullable, - } - for parameter in test.parameter_schema - ], - "parameters": [ - { - "name": name, - "type": parameter_types[name], - "value": _typed_value_payload(value), - } - for name, value in test.parameter_values - ], - } - ) - return payload - - -def _sql_test_cte_payloads(ctes: tuple[CompileSqlTestCte, ...]) -> list[dict[str, str]]: - return [{"name": cte.name, "sql": cte.sql_body} for cte in ctes] - - -def _sql_scenario_payloads(project: CompiledProject) -> list[dict[str, object]]: - return [_sql_scenario_payload(scenario) for scenario in project.sql_scenarios] - - -def _sql_scenario_payload(scenario: CompiledSqlScenario) -> dict[str, object]: - description: object | None = scenario.scenario_file.header_values.get("description") - return { - "source_path": (scenario.source_path or scenario.scenario_file.relative_path).as_posix(), - "ownership_root": ( - scenario.ownership_root or scenario.scenario_file.ownership_root - ).as_posix(), - "name": scenario.name, - "description": description if isinstance(description, str) else None, - "expected_model_names": list(scenario.expected_model_names), - "assertion_names": list(scenario.assertion_names), - "assertion_target_model_names": list(scenario.assertion_target_model_names), - "target_model_names": list(scenario.target_model_names), - } - - -def _model_payload( - *, - model: CompiledModel, - compiled_audit_count: int, - targeting_test_count: int, - dialect: str, - include_type_proof: bool, -) -> dict[str, object]: - schema_audit_count: int = 0 - columns: list[dict[str, object]] = [] - if model.schema_entry is not None: - inferred_by_name: dict[str, InferredColumn] = { - column.name: column for column in (model.inferred_columns or ()) - } - schema_audit_count = len(model.schema_entry.audits) + sum( - len(column.audits) for column in model.schema_entry.columns - ) - for column in model.schema_entry.columns: - payload: dict[str, object] = { - "name": column.name, - "type": column.type or "", - "nullable": column.nullable, - "audit_count": len(column.audits), - } - if include_type_proof: - payload["type_proven"] = bool( - column.type - and (inferred := inferred_by_name.get(column.name)) is not None - and inferred.type - and types_equal( - left=column.type, - right=inferred.type, - dialect=dialect, - ) - ) - columns.append(payload) - dynamic_columns: list[dict[str, object]] = [] - dynamic_columns_proven: bool = False - if model.schema_entry is not None and model.schema_entry.dynamic_columns: - proof: DynamicColumnContractProof | None = model.dynamic_column_contract - dynamic_columns_proven = bool(proof is not None and proof.output_proven) - proof_types: dict[str, str | None] = ( - {family.name.casefold(): family.inferred_type for family in proof.families} - if proof is not None - else {} - ) - for family in model.schema_entry.dynamic_columns: - inferred_type: str | None = proof_types.get(family.name.casefold()) - dynamic_columns.append( - { - "name": family.name, - "pivot_column": family.pivot_column, - "value_column": family.value_column, - "aggregate": family.aggregate, - "type": family.type, - "name_pattern": family.name_pattern, - "type_proven": bool( - include_type_proof - and inferred_type - and types_equal( - left=family.type, - right=inferred_type, - dialect=dialect, - ) - ), - } - ) - return { - "name": model.name, - "relative_path": model.relative_path.as_posix(), - "query_sql": model.query_sql, - "authored_sql": model.authored_query_sql or model.authored_sql, - "config": model.config.values, - "authored_config_keys": list(model.config.model_header_keys), - "logical_schema": model.config.layer_schema, - "references": [ - { - "ref_kind": str(reference.ref_kind), - "ref_name": reference.ref_name, - "ref_package": reference.ref_package, - } - for reference in model.references - ], - "columns": columns, - "dynamic_columns": dynamic_columns, - "dynamic_columns_proven": dynamic_columns_proven, - "bare_dynamic_pivot": bool( - model.dynamic_column_contract is not None - and model.dynamic_column_contract.bare_dynamic_pivot - ), - "enum_columns": list(model.enum_columns), - "enum_declarations": [ - _enum_payload(declaration) for declaration in model.enum_declarations - ], - "constant_declarations": [ - _constant_payload(declaration) for declaration in model.constant_declarations - ], - "declared_audit_count": max(schema_audit_count, compiled_audit_count), - "targeting_test_count": targeting_test_count, - } - - -def _enum_payload(declaration: EnumDeclaration) -> dict[str, object]: - return { - "name": declaration.name, - "relative_path": declaration.relative_path.as_posix(), - "members": [{"name": member.name, "value": member.value} for member in declaration.members], - } - - -def _constant_payload(declaration: ConstantDeclaration) -> dict[str, object]: - return { - "name": declaration.name, - "relative_path": declaration.relative_path.as_posix(), - "members": [], - "value": _typed_value_payload(declaration.value), - "value_type": declaration.logical_type.display_name, - "render_as": declaration.render_as.value if declaration.render_as is not None else None, - } - - -def _typed_value_payload(value: SqlValue) -> object: - if value.kind == SqlValueKind.DECIMAL: - return str(cast(Decimal, value.value)) - if value.kind in { - SqlValueKind.STRING, - SqlValueKind.INTEGER, - SqlValueKind.BOOLEAN, - SqlValueKind.FLOAT, - SqlValueKind.NULL, - }: - return value.value - if value.kind in {SqlValueKind.LIST, SqlValueKind.SET}: - return [_typed_value_payload(item) for item in cast(tuple[SqlValue, ...], value.value)] - return { - key: _typed_value_payload(item) - for key, item in cast(tuple[tuple[str, SqlValue], ...], value.value) - } - - def custom_rule_payloads( *, catalogue: tuple[Rule, ...], project_dir: Path ) -> list[dict[str, object]]: diff --git a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py index bd2b1853fa..2fbbda280c 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/rule_harness.py @@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project import compile_project @@ -39,7 +39,7 @@ def run_rule_case(*, rule: RuleCheck | Rule, test_case: RuleCase) -> RuleResult: with TemporaryDirectory(prefix="sqlbuild-rule-") as temporary: root: Path = Path(temporary) _ = _write_rule_project(root=root, rule=resolved, test_case=test_case) - project: CompiledProject = _compile_rule_project(root=root) + project: CompiledProjectView = _compile_rule_project(root=root) config: RulesConfig = load_rules_config(root) result: RulesResult = evaluate( project=project, @@ -107,7 +107,7 @@ def _toml_value(*, value: object) -> str: raise RuleUsageError(f"RuleCase config value {value!r} cannot be represented as a rule option") -def _compile_rule_project(*, root: Path) -> CompiledProject: +def _compile_rule_project(*, root: Path) -> CompiledProjectView: discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=root) adapter: DuckDbAdapter = DuckDbAdapter() return compile_project(discovered_inputs=discovered, adapter=adapter) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py index 1455afdfa1..5c3699b769 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/ruleset.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Any -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( @@ -20,7 +20,7 @@ def evaluate_project( *, - project: CompiledProject, + project: CompiledProjectView, config: RulesConfig, project_dir: Path, dialect: str = "generic", diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py index 9e9c501d90..0f40861c9e 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_evaluation.py @@ -9,7 +9,7 @@ from itertools import chain from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine._helpers.host.module_state import rule_namespaces from sqlbuild.rule_engine._helpers.host.tracked_facts import tracked_fact_views from sqlbuild.rule_engine.classes.fact_reads import FactReads @@ -47,7 +47,7 @@ @dataclass(frozen=True) class _PassInputs: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project_dir: Path selected_rules: tuple[Rule, ...] @@ -60,7 +60,7 @@ class _PassInputs: def evaluate_custom_rules( *, - project: CompiledProject, + project: CompiledProjectView, config: RulesConfig, project_dir: Path, selected_rules: tuple[Rule, ...], @@ -78,7 +78,7 @@ def evaluate_custom_rules( ) if not custom_rules: return CustomRuleRun(evaluations=(), untracked_codes=frozenset()) - models: tuple[CompiledModel, ...] = tuple( + models: tuple[CompiledModelView, ...] = tuple( sorted(project.models, key=lambda item: item.relative_path.as_posix()) ) facts: RuleFactViews = build_rule_fact_views( @@ -105,7 +105,7 @@ def evaluate_custom_rules( ) def run_pass( - *, rules: tuple[Rule, ...], subjects: tuple[CompiledModel, ...], active: _ReadTracker + *, rules: tuple[Rule, ...], subjects: tuple[CompiledModelView, ...], active: _ReadTracker ) -> list[CustomRuleEvaluation]: return _evaluate_pass(inputs=inputs, rules=rules, models=subjects, tracker=active) @@ -183,11 +183,11 @@ def _evaluate_pass( *, inputs: _PassInputs, rules: tuple[Rule, ...], - models: tuple[CompiledModel, ...], + models: tuple[CompiledModelView, ...], tracker: _ReadTracker, ) -> list[CustomRuleEvaluation]: evaluations: list[CustomRuleEvaluation] = [] - project: CompiledProject = inputs.project + project: CompiledProjectView = inputs.project plan: CustomRulePlan | None = inputs.plan guard: RuntimeGuard | None = inputs.guard context_facts: RuleFactViews = ( diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index b6223e5a55..4923494533 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -15,6 +15,7 @@ from typing import Any, BinaryIO from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload from sqlbuild.rule_engine._helpers.host.custom_evaluation import evaluate_custom_rules @@ -118,7 +119,7 @@ def main() -> int: def _decode_inputs( *, payload: dict[str, Any], cancel_marker: Path | None -) -> tuple[CompiledProject, RulesConfig]: +) -> tuple[CompiledProjectView, RulesConfig]: project_dir: Path = Path(str(payload["project_dir"])).resolve() input_root: Path = (project_dir / "target" / "rules-cache" / "host-inputs").resolve() input_path: Path = Path(str(payload["project_pickle_path"])) diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py index cff549221a..654e0851cc 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py @@ -18,6 +18,8 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.rule_engine._helpers.host.host_capacity import available_cores from sqlbuild.rule_engine.constants import ( CUSTOM_HOST_ABANDONED_SUFFIX, @@ -44,7 +46,7 @@ def run_custom_hosts( *, - project: CompiledProject, + project: CompiledProjectView, config: RulesConfig, project_dir: Path, dialect: str, @@ -96,10 +98,10 @@ def run_custom_hosts( ) -def _write_inputs(*, path: Path, project: CompiledProject, config: RulesConfig) -> None: +def _write_inputs(*, path: Path, project: CompiledProjectView, config: RulesConfig) -> None: """Publish the project payload atomically; hosts started earlier wait for it to appear.""" - payload: CompiledProject = host_payload_project(project) + payload: CompiledProjectView = host_payload_project(project) with tempfile.NamedTemporaryFile( dir=path.parent, prefix="writing-", suffix=".pickle", delete=False ) as handle: @@ -113,25 +115,38 @@ def _write_inputs(*, path: Path, project: CompiledProject, config: RulesConfig) partial.replace(path) -def host_payload_project(project: CompiledProject) -> CompiledProject: +def host_payload_project(project: CompiledProjectView) -> CompiledProject: """Return the project hosts decode, without model state no Rule fact reads.""" - return replace(project, models=tuple(map(_payload_model, project.models))) + return replace( + compiled_project_record(project), + models=tuple(map(_payload_model, project.models)), + native_project=None, + ) -def _payload_model(model: CompiledModel) -> CompiledModel: +def _payload_model(model: CompiledModelView) -> CompiledModelView: """Drop model state no Rule fact reads, so hosts decode less of every model.""" - return replace( - model, - references=(), - fast_lineage_columns=None, - authored_query_sql="", - output_column_locations={}, - macro_deps=(), - enum_columns={}, - binding_diagnostics=(), - unchecked_output_columns=frozenset(), + return CompiledModel( + key=model.key, + deps=model.deps, + name=model.name, + relative_path=model.relative_path, + query_sql=model.query_sql, + config=model.config, + destination=model.destination, + schema_entry=model.schema_entry, + inferred_columns=model.inferred_columns, + fast_lineage_has_star=model.fast_lineage_has_star, + fast_lineage_star_resolved=model.fast_lineage_star_resolved, + authored_sql=model.authored_sql, + extract_implicit_alias_columns=model.extract_implicit_alias_columns, + enum_declarations=model.enum_declarations, + constant_declarations=model.constant_declarations, + binding_validated=model.binding_validated, + dynamic_column_contract=model.dynamic_column_contract, + rejected_sql_analysis_opt_out=model.rejected_sql_analysis_opt_out, ) diff --git a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py index 6864c9ca68..9cdd9567fa 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py +++ b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py @@ -6,7 +6,8 @@ from pathlib import Path from typing import Any -from sqlbuild.compiler.compile.models import CompiledAudit, CompiledSqlTest, InferredColumn +from sqlbuild.compiler.compile.models import CompiledSqlTest, InferredColumn +from sqlbuild.compiler.compile.types import CompiledAuditView from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_error_digest, fact_value_digest from sqlbuild.rule_engine.classes.fact_reads import FactReads @@ -244,11 +245,11 @@ def for_model(self, model: Model) -> tuple[CompiledSqlTest, ...]: class _TrackedAuditFacts(_TrackedView): __slots__ = () - def all(self) -> tuple[CompiledAudit, ...]: + def all(self) -> tuple[CompiledAuditView, ...]: self._tracked_reads.record(key=(FACT_AUDITS_ALL,)) return self._tracked_view.all() - def for_model(self, model: Model) -> tuple[CompiledAudit, ...]: + def for_model(self, model: Model) -> tuple[CompiledAuditView, ...]: self._tracked_reads.record_model_name(fact=FACT_AUDITS_FOR_MODEL, model=model) return self._tracked_view.for_model(model) diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py deleted file mode 100644 index d1ceddcb9f..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Reuse the complete built-in rules response for an identical native request.""" - -from __future__ import annotations - -import hashlib -import os -import threading -from pathlib import Path -from typing import Any - -import orjson - -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path -from sqlbuild.rule_engine.constants import NATIVE_RULES_MEMO_FILE, NATIVE_RULES_MEMO_VERSION - - -def native_payload_digests(payloads: list[bytes]) -> list[str]: - """Digest each exact encoded model payload; the digest keys that model's cached findings.""" - - return [hashlib.blake2b(payload, digest_size=32).hexdigest() for payload in payloads] - - -def native_request_identity(*, request_json: bytes, model_digests: list[str]) -> str: - """Digest the model-free request plus ordered digests of each exact model payload.""" - - digest: Any = hashlib.blake2b(digest_size=32) - digest.update(NATIVE_RULES_MEMO_VERSION.encode()) - digest.update(b"\0") - digest.update(compiled_code_identity().encode()) - digest.update(b"\0") - digest.update(request_json) - digest.update(b"\0") - digest.update(len(model_digests).to_bytes(8, "little")) - digest.update("\0".join(model_digests).encode()) - return digest.hexdigest() - - -def read_native_response(*, project_dir: Path, identity: str) -> str | None: - """Return the stored response for this exact request identity, if any.""" - - try: - payload: object = orjson.loads( - rules_bulk_cache_path( - project_dir=project_dir, file_name=NATIVE_RULES_MEMO_FILE - ).read_bytes() - ) - except (OSError, orjson.JSONDecodeError): - return None - if not isinstance(payload, dict) or payload.get("identity") != identity: - return None - response: object = payload.get("response") - return response if isinstance(response, str) else None - - -def write_native_response(*, project_dir: Path, identity: str, response: str) -> None: - """Atomically replace the stored response with the latest evaluated request.""" - - path: Path = rules_bulk_cache_path(project_dir=project_dir, file_name=NATIVE_RULES_MEMO_FILE) - path.parent.mkdir(parents=True, exist_ok=True) - temporary: Path = path.with_suffix(f".tmp-{os.getpid()}-{threading.get_ident()}") - temporary.write_bytes(orjson.dumps({"identity": identity, "response": response})) - temporary.replace(path) diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py new file mode 100644 index 0000000000..fcb2e0f08c --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py @@ -0,0 +1,247 @@ +"""Hand the native rules request builder rows read from the Python compiled project.""" + +from __future__ import annotations + +from dataclasses import asdict +from functools import partial +from pathlib import Path + +import orjson + +import sqlbuild._native as _native +from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.compiler.compile.models import ( + CompiledModelSqlTestPayload, + CompiledSqlScenario, + CompiledSqlTest, + CompileSqlTestCte, +) +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_project_facts import compiled_project_facts +from sqlbuild.compiler.compiled_project.main.declaration_rows import declaration_rows +from sqlbuild.compiler.compiled_project.main.plain_typed_value import plain_typed_value +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path +from sqlbuild.rule_engine.constants import ( + NATIVE_RULES_CACHE_FILE, + NATIVE_RULES_MEMO_FILE, + RULES_NATIVE_API_VERSION, +) +from sqlbuild.rule_engine.exceptions import RulesError +from sqlbuild.rule_engine.models import Finding, NativeRulesEvaluation, RulesConfig +from sqlbuild.rule_engine.types import FindingRow + + +def build_rules_request( + *, + project: CompiledProjectView, + config: RulesConfig, + project_dir: Path, + dialect: str, + initial_findings: tuple[Finding, ...], + custom_payloads: list[dict[str, object]], + include_type_proof: bool, +) -> _native.NativeRulesRequest: + """Build the built-in rules request natively from the compiled project's rows.""" + + resolved: Path = project_dir.resolve() + header: dict[str, object] = { + "version": RULES_NATIVE_API_VERSION, + "project_dir": str(resolved), + "dialect": dialect, + "config": asdict(config), + "scope_index": scope_metadata_projection(index=project.scope_index), + "defer_suppressions": True, + "custom_rules": custom_payloads, + "rules_cache_path": str( + rules_bulk_cache_path(project_dir=resolved, file_name=NATIVE_RULES_CACHE_FILE) + ), + } + public_enums, public_constants = declaration_rows( + enums=tuple(project.public_enums.values()), + constants=tuple(project.public_constants.values()), + ) + project_rows: dict[str, object] = { + "header_json": _encoded(header), + "memo": ( + ( + compiled_code_identity(), + str( + rules_bulk_cache_path(project_dir=project_dir, file_name=NATIVE_RULES_MEMO_FILE) + ), + ) + if config.cache.enabled + else None + ), + "sql_analysis_enabled": project.settings.sql_analysis, + "include_type_proof": include_type_proof, + "attached_audit_targets": [ + audit.attached_target_name + for audit in project.audits + if audit.attached_target_name is not None + ], + "public_enums": public_enums, + "public_constants": public_constants, + "initial_findings": [finding_row(finding) for finding in initial_findings], + "types_equal": partial(types_equal, dialect=dialect), + } + try: + request: _native.NativeRulesRequest + counts: tuple[int, int, int] + request, counts = _native.build_rules_request( + project_rows, + (compiled_project_facts(project), [model.name for model in project.models]), + [_sql_test_row(test) for test in project.sql_tests], + [_scenario_row(scenario) for scenario in project.sql_scenarios], + ) + except (ValueError, TypeError) as error: + raise RulesError(str(error)) from error + for kind, units in zip(("models", "sql_tests", "sql_scenarios"), counts, strict=True): + report_native_answer(stage=NativeStage.RULES_REQUEST, kind=kind, units=units) + return request + + +def evaluate_rules_request(request: _native.NativeRulesRequest) -> NativeRulesEvaluation: + """Answer the request from the response memo or evaluate it, decoding findings natively.""" + + try: + faults, selected_codes, evaluated, hits, misses, built_in_ms, reused = ( + _native.evaluate_rules_request(request) + ) + except (ValueError, TypeError) as error: + raise RulesError(str(error)) from error + return NativeRulesEvaluation( + findings=tuple(finding_from_row(row) for row in faults), + selected_codes=tuple(selected_codes), + evaluated_models=evaluated, + cache_hits=hits, + cache_misses=misses, + built_in_ms=built_in_ms, + reused=reused, + ) + + +def finalize_rules_rows( + *, + config: RulesConfig, + project_dir: Path, + evaluated_codes: tuple[str, ...], + findings: tuple[Finding, ...], +) -> tuple[Finding, ...]: + """Apply the exception policy natively to finding rows.""" + + try: + rows: list[FindingRow] = _native.finalize_rule_findings_rows( + { + "project_dir": str(project_dir.resolve()), + "config_json": _encoded(asdict(config)), + "evaluated_codes": list(evaluated_codes), + "findings": [finding_row(finding) for finding in findings], + } + ) + except (ValueError, TypeError) as error: + raise RulesError(str(error)) from error + return tuple(finding_from_row(row) for row in rows) + + +def finding_row(finding: Finding) -> FindingRow: + """One finding as the native boundary's row.""" + + return ( + finding.unevaluated, + finding.code, + finding.path.as_posix(), + finding.line, + finding.column, + finding.message, + finding.remediation, + ) + + +def finding_from_row(row: FindingRow) -> Finding: + """One native finding row as a finding.""" + + unevaluated, code, path, line, column, message, remediation = row + return Finding( + code=code, + path=Path(path), + line=line, + column=column, + message=message, + remediation=remediation, + unevaluated=unevaluated, + ) + + +def _encoded(value: object) -> bytes: + try: + return orjson.dumps(value, option=orjson.OPT_SORT_KEYS, default=str) + except orjson.JSONEncodeError as error: + raise RulesError(str(error)) from error + + +def _sql_test_row(test: CompiledSqlTest) -> tuple[object, ...]: + payload: object = ( + ( + _cte_rows(test.payload.authored_ctes), + _cte_rows(test.payload.expected_ctes), + _cte_rows(test.payload.assertion_ctes), + bool(test.payload.macro_mocks), + bool(test.payload.model_query_overrides), + ) + if isinstance(test.payload, CompiledModelSqlTestPayload) + else None + ) + case: object = ( + None + if test.case_name is None + else ( + test.parent_name, + test.case_name, + test.case_index, + test.case_fingerprint, + [ + (parameter.name, parameter.value_type.value, parameter.nullable) + for parameter in test.parameter_schema + ], + [(name, plain_typed_value(value)) for name, value in test.parameter_values], + ) + ) + return ( + (test.source_path or test.test_file.relative_path).as_posix(), + (test.ownership_root or test.test_file.ownership_root).as_posix(), + test.block_index or test.test_block.test_index, + test.name, + test.explicit_name, + test.mode.value, + ( + list(test.expected_model_names), + list(test.assertion_names), + list(test.assertion_target_model_names), + list(test.target_model_names), + ), + [(resource.kind.value, resource.name) for resource in test.tested_resources], + payload, + case, + ) + + +def _cte_rows(ctes: tuple[CompileSqlTestCte, ...]) -> list[tuple[str, str]]: + return [(cte.name, cte.sql_body) for cte in ctes] + + +def _scenario_row(scenario: CompiledSqlScenario) -> tuple[object, ...]: + description: object | None = scenario.scenario_file.header_values.get("description") + return ( + (scenario.source_path or scenario.scenario_file.relative_path).as_posix(), + (scenario.ownership_root or scenario.scenario_file.ownership_root).as_posix(), + scenario.name, + description if isinstance(description, str) else None, + list(scenario.expected_model_names), + list(scenario.assertion_names), + list(scenario.assertion_target_model_names), + list(scenario.target_model_names), + ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 6b4ccd8567..53f68d617a 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -16,16 +16,22 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledSqlExpansion, - CompileProjectInputs, DeclarationScopeBuild, SqlExpansionContext, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompileInputsView, + ModelInput, +) +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.main.report_native_answer import report_native_answer +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.constants import ( @@ -33,9 +39,6 @@ DEFAULT_MAX_RANKING_ORDER_BY, MAX_LITERAL_LENGTH_THRESHOLD, MAX_RANKING_ORDER_BY_THRESHOLD, - RELATION_KIND_REF, - RELATION_KIND_SEED, - RELATION_KIND_SOURCE, SAFE_RULE_FIX_CODES, TEMPLATE_INTERPOLATION_START, ) @@ -142,7 +145,7 @@ def evaluate_rules( model_paths: frozenset[str] | None = _selected_model_paths( graph=graph, selected_keys=selected_keys ) - selected_project: CompiledProject = _selected_project(graph=graph, model_paths=model_paths) + selected_project: CompiledProjectView = _selected_project(graph=graph, model_paths=model_paths) selected_config: RulesConfig = replace( _selection_rules_config( config=effective_config, @@ -241,7 +244,7 @@ def evaluate_rules( def _prepare_sql_rule_inputs( *, rules: tuple[Rule, ...], - project: CompiledProject, + project: CompiledProjectView, dialect: str, project_dir: Path, collect_files: bool, @@ -279,7 +282,7 @@ def _config_references_custom_rules(config: RulesConfig) -> bool: def prepare_sql_rules( - *, inputs: CompileProjectInputs, executor: Executor, dialect: str, stop: Event + *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event ) -> PreparedSqlLint | None: project_dir: Path | None = inputs.discovered_inputs.project_dir if project_dir is None or any(diagnostic.is_error for diagnostic in inputs.diagnostics): @@ -305,44 +308,37 @@ def prepare_sql_rules( thresholds=config.thresholds, allow_suppressions=config.allow_exceptions, relation_keys=declared_relation_keys( - relations=( - *( - ( - RELATION_KIND_REF, - model.model_file.file_path.stem, - model.schema_entry, - model.config.values, - ) - for model in inputs.model_inputs - ), - *( - (RELATION_KIND_SEED, seed.schema_entry.name, seed.schema_entry, {}) - for seed in inputs.seed_inputs - ), - *( - ( - RELATION_KIND_SOURCE, - source.source_entry.name, - source.source_entry, - {}, - ) - for source in inputs.source_inputs - ), - ) + model_rows=(model.relation_key_row for model in inputs.model_inputs), + seeds=(seed.schema_entry for seed in inputs.seed_inputs), + sources=(source.source_entry for source in inputs.source_inputs), ), ), discovered_inputs=inputs.discovered_inputs, static_declaration_scope=inputs.declaration_scope, - compiled_expansions={ - model.model_file.file_path: model.sql_expansion - for model in inputs.model_inputs - if model.sql_expansion is not None - }, + compiled_expansions=early_lint_expansions(inputs.model_inputs), stop=stop, ), ) +def early_lint_expansions( + model_inputs: tuple[ModelInput, ...], +) -> dict[Path, CompiledSqlExpansion]: + """Each model's compile expansion for early lint, whole when the compile handed one over.""" + + expansions: dict[Path, CompiledSqlExpansion] = {} + for model in model_inputs: + expansion: CompiledSqlExpansion | None = model.lint_expansion or model.sql_expansion + if expansion is not None: + expansions[model.model_file.file_path] = expansion + report_native_answer( + stage=NativeStage.COMPILE_LINT_INPUTS, + kind="lint_expansions", + units=sum(model.lint_expansion is not None for model in model_inputs), + ) + return expansions + + def _prepare_sql_lint( *, project_dir: Path, @@ -375,7 +371,7 @@ def _selection_rules_config( *, config: RulesConfig, model_paths: frozenset[str] | None, - project: CompiledProject, + project: CompiledProjectView, ) -> RulesConfig: if model_paths is None: return config @@ -408,11 +404,11 @@ def _selected_sql_codes(config: RulesConfig) -> tuple[str, ...]: def _selected_project( *, graph: ProjectGraph, model_paths: frozenset[str] | None -) -> CompiledProject: +) -> CompiledProjectView: if model_paths is None: return graph.project - return replace( - graph.project, + return project_with( + project=graph.project, models=tuple( model for model in graph.project.models if model.relative_path.as_posix() in model_paths ), @@ -493,7 +489,7 @@ def _run_sql_rules( rules: tuple[Rule, ...], project_dir: Path, discovered_inputs: DiscoveredProjectInputs, - project: CompiledProject, + project: CompiledProjectView, lint_config: LintConfig, selected_model_paths: frozenset[str] | None, cache_enabled: bool, @@ -504,7 +500,7 @@ def _run_sql_rules( codes: tuple[str, ...] = _sql_rule_codes(rules) if not codes: return _SqlRulesEvaluation(findings=(), cache_hits=0, cache_misses=0) - models_by_path: dict[str, CompiledModel] = _sql_rule_models(project) + models_by_path: dict[str, CompiledModelView] = _sql_rule_models(project) bucket: dict[str, dict[str, object]] = ( _read_sql_rule_cache(project_dir) if cache_enabled else {} ) @@ -721,7 +717,7 @@ def _sql_rule_codes(rules: tuple[Rule, ...]) -> tuple[str, ...]: return tuple(rule.code for rule in rules if rule.code.startswith("SQBRSQL")) -def _sql_rule_models(project: CompiledProject) -> dict[str, CompiledModel]: +def _sql_rule_models(project: CompiledProjectView) -> dict[str, CompiledModelView]: return { model.relative_path.as_posix(): model for model in project.models @@ -730,7 +726,7 @@ def _sql_rule_models(project: CompiledProject) -> dict[str, CompiledModel]: def _sql_model_rule_identities( - *, rules: tuple[Rule, ...], project: CompiledProject, dialect: str + *, rules: tuple[Rule, ...], project: CompiledProjectView, dialect: str ) -> dict[str, str]: codes: tuple[str, ...] = _sql_rule_codes(rules) if not codes: @@ -741,7 +737,7 @@ def _sql_model_rule_identities( } -def _sql_rule_identity(*, model: CompiledModel, codes: tuple[str, ...], dialect: str) -> str: +def _sql_rule_identity(*, model: CompiledModelView, codes: tuple[str, ...], dialect: str) -> str: digest: Any = hashlib.blake2b(digest_size=32) digest.update(_SQL_RULE_CACHE_VERSION.encode()) digest.update(_RULES_BUILD_IDENTITY.encode()) @@ -798,12 +794,18 @@ def _sql_rule_cache_path(project_dir: Path) -> Path: def _read_sql_rule_cache(project_dir: Path) -> dict[str, dict[str, object]]: + """Return usable entries so incompatible builds cannot suppress early lint.""" + path: Path = _sql_rule_cache_path(project_dir) try: payload: object = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError): return {} - if not isinstance(payload, dict) or payload.get("version") != _SQL_RULE_CACHE_VERSION: + if ( + not isinstance(payload, dict) + or payload.get("version") != _SQL_RULE_CACHE_VERSION + or payload.get("build_identity") != _RULES_BUILD_IDENTITY + ): return {} entries: object = payload.get("entries") if not isinstance(entries, dict): @@ -844,7 +846,11 @@ def _write_sql_rule_cache(*, project_dir: Path, bucket: dict[str, dict[str, obje temporary: Path = path.with_suffix(f".tmp-{os.getpid()}") temporary.write_text( json.dumps( - {"version": _SQL_RULE_CACHE_VERSION, "entries": bucket}, + { + "version": _SQL_RULE_CACHE_VERSION, + "build_identity": _RULES_BUILD_IDENTITY, + "entries": bucket, + }, sort_keys=True, separators=(",", ":"), ), diff --git a/src/sqlbuild/rule_engine/classes/audit_facts.py b/src/sqlbuild/rule_engine/classes/audit_facts.py index e703a20a87..8a9c0ba1cc 100644 --- a/src/sqlbuild/rule_engine/classes/audit_facts.py +++ b/src/sqlbuild/rule_engine/classes/audit_facts.py @@ -1,21 +1,26 @@ """Audit facts for custom rules.""" -from sqlbuild.compiler.compile.models import CompiledAudit, CompiledProject +from functools import cached_property + +from sqlbuild.compiler.compile.models import CompiledAudit +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_audit_record import compiled_audit_record from sqlbuild.rule_engine.models import Model class AuditFacts: """Compiled audits indexed by their model subject.""" - def __init__(self, *, project: CompiledProject) -> None: - self._audits: tuple[CompiledAudit, ...] = project.audits + def __init__(self, *, project: CompiledProjectView) -> None: + self._audits: tuple[CompiledAudit, ...] = tuple(map(compiled_audit_record, project.audits)) + + @cached_property + def _by_model(self) -> dict[str, tuple[CompiledAudit, ...]]: by_model: dict[str, list[CompiledAudit]] = {} for audit in self._audits: if audit.attached_target_name is not None: by_model.setdefault(audit.attached_target_name, []).append(audit) - self._by_model: dict[str, tuple[CompiledAudit, ...]] = { - name: tuple(audits) for name, audits in by_model.items() - } + return {name: tuple(audits) for name, audits in by_model.items()} def all(self) -> tuple[CompiledAudit, ...]: return self._audits diff --git a/src/sqlbuild/rule_engine/classes/column_facts.py b/src/sqlbuild/rule_engine/classes/column_facts.py index 99899fd897..f6b62b392a 100644 --- a/src/sqlbuild/rule_engine/classes/column_facts.py +++ b/src/sqlbuild/rule_engine/classes/column_facts.py @@ -1,19 +1,24 @@ """Column facts for custom rules.""" -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, InferredColumn +from functools import cached_property + +from sqlbuild.compiler.compile.models import InferredColumn +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.models import Model class ColumnFacts: """Declared and compiler-inferred output columns.""" - def __init__(self, *, project: CompiledProject) -> None: - self._models: dict[object, CompiledModel] = { - model.relative_path: model for model in project.models - } + def __init__(self, *, project: CompiledProjectView) -> None: + self._project: CompiledProjectView = project + + @cached_property + def _models(self) -> dict[object, CompiledModelView]: + return {model.relative_path: model for model in self._project.models} def declared(self, model: Model) -> tuple[object, ...]: - compiled: CompiledModel = self._models[model.path] + compiled: CompiledModelView = self._models[model.path] return () if compiled.schema_entry is None else compiled.schema_entry.columns def inferred(self, model: Model) -> tuple[InferredColumn, ...] | None: diff --git a/src/sqlbuild/rule_engine/classes/contract_facts.py b/src/sqlbuild/rule_engine/classes/contract_facts.py index 2c8e4076b2..f5e757925a 100644 --- a/src/sqlbuild/rule_engine/classes/contract_facts.py +++ b/src/sqlbuild/rule_engine/classes/contract_facts.py @@ -1,6 +1,6 @@ """Contract facts for custom rules.""" -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.constants import CONTRACT_ENFORCED from sqlbuild.rule_engine.models import Model @@ -8,8 +8,8 @@ class ContractFacts: """Compiler-resolved model contract facts.""" - def __init__(self, *, project: CompiledProject) -> None: - self._models: dict[object, CompiledModel] = { + def __init__(self, *, project: CompiledProjectView) -> None: + self._models: dict[object, CompiledModelView] = { model.relative_path: model for model in project.models } diff --git a/src/sqlbuild/rule_engine/classes/declaration_facts.py b/src/sqlbuild/rule_engine/classes/declaration_facts.py index 67310208f4..c410d315b0 100644 --- a/src/sqlbuild/rule_engine/classes/declaration_facts.py +++ b/src/sqlbuild/rule_engine/classes/declaration_facts.py @@ -1,13 +1,13 @@ """Declaration facts for custom rules.""" -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration class DeclarationFacts: """Compiler-resolved public and model-scoped declarations.""" - def __init__(self, *, project: CompiledProject) -> None: + def __init__(self, *, project: CompiledProjectView) -> None: self.public_enums: tuple[EnumDeclaration, ...] = tuple(project.public_enums.values()) self.public_constants: tuple[ConstantDeclaration, ...] = tuple( project.public_constants.values() diff --git a/src/sqlbuild/rule_engine/classes/early_sql_lint.py b/src/sqlbuild/rule_engine/classes/early_sql_lint.py index 2da7272ec8..1e0cd3d9c7 100644 --- a/src/sqlbuild/rule_engine/classes/early_sql_lint.py +++ b/src/sqlbuild/rule_engine/classes/early_sql_lint.py @@ -7,7 +7,7 @@ from threading import Event from types import TracebackType -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.rule_engine.main._prepare_sql_rules import prepare_sql_rules from sqlbuild.rule_engine.models import PreparedSqlLint, SqlExpansionReuse @@ -33,7 +33,7 @@ def __exit__( ) -> None: _ = self.stop() - def start(self, *, inputs: CompileProjectInputs, dialect: str) -> None: + def start(self, *, inputs: CompileInputsView, dialect: str) -> None: if self.enabled: self.preparation = prepare_sql_rules( inputs=inputs, diff --git a/src/sqlbuild/rule_engine/classes/graph_facts.py b/src/sqlbuild/rule_engine/classes/graph_facts.py index 4c293e0f9e..98df4f64ec 100644 --- a/src/sqlbuild/rule_engine/classes/graph_facts.py +++ b/src/sqlbuild/rule_engine/classes/graph_facts.py @@ -1,6 +1,8 @@ """Dependency graph facts for custom rules.""" -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from functools import cached_property + +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import public_model from sqlbuild.rule_engine.models import Model @@ -8,23 +10,39 @@ class GraphFacts: """Compiler-resolved model dependencies and dependents.""" - def __init__(self, *, project: CompiledProject) -> None: - self._compiled_by_path: dict[object, CompiledModel] = { - model.relative_path: model for model in project.models - } - self._by_key: dict[object, Model] = { - model.key: public_model(model) for model in project.models - } + def __init__( + self, *, project: CompiledProjectView, models: tuple[Model, ...] | None = None + ) -> None: + self._project: CompiledProjectView = project + self._models: tuple[Model, ...] | None = models + + @cached_property + def _compiled_by_path(self) -> dict[object, CompiledModelView]: + return {model.relative_path: model for model in self._project.models} + + @cached_property + def _by_key(self) -> dict[object, Model]: + model_views: tuple[Model, ...] = ( + tuple(public_model(item) for item in self._project.models) + if self._models is None + else self._models + ) + public_by_path: dict[object, Model] = {item.path: item for item in model_views} + return {model.key: public_by_path[model.relative_path] for model in self._project.models} + + @cached_property + def _downstream(self) -> dict[object, list[Model]]: downstream: dict[object, list[Model]] = {} - for model in project.models: + for model in self._project.models: + subject: Model = self._by_key[model.key] for dependency in model.deps: - downstream.setdefault(dependency, []).append(public_model(model)) - self._downstream: dict[object, list[Model]] = downstream + downstream.setdefault(dependency, []).append(subject) + return downstream def dependencies(self, model: Model) -> tuple[Model, ...]: - compiled: CompiledModel = self._compiled_by_path[model.path] + compiled: CompiledModelView = self._compiled_by_path[model.path] return tuple(self._by_key[key] for key in compiled.deps if key in self._by_key) def dependents(self, model: Model) -> tuple[Model, ...]: - compiled: CompiledModel = self._compiled_by_path[model.path] + compiled: CompiledModelView = self._compiled_by_path[model.path] return tuple(sorted(self._downstream.get(compiled.key, ()), key=lambda item: item.name)) diff --git a/src/sqlbuild/rule_engine/classes/model_test_facts.py b/src/sqlbuild/rule_engine/classes/model_test_facts.py index 2dbeb13462..395bff6d35 100644 --- a/src/sqlbuild/rule_engine/classes/model_test_facts.py +++ b/src/sqlbuild/rule_engine/classes/model_test_facts.py @@ -1,24 +1,27 @@ """SQL test facts for custom rules.""" -from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest -from sqlbuild.compiler.compile.types import SqlTestMode +from functools import cached_property + +from sqlbuild.compiler.compile.models import CompiledSqlTest +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from sqlbuild.rule_engine.models import Model class TestFacts: """Compiled SQL tests indexed by their model subject.""" - def __init__(self, *, project: CompiledProject) -> None: + def __init__(self, *, project: CompiledProjectView) -> None: self._tests: tuple[CompiledSqlTest, ...] = project.sql_tests + + @cached_property + def _by_model(self) -> dict[str, tuple[CompiledSqlTest, ...]]: by_model: dict[str, list[CompiledSqlTest]] = {} for test in self._tests: if test.mode is not SqlTestMode.MODEL: continue for model_name in dict.fromkeys(test.target_model_names): by_model.setdefault(model_name, []).append(test) - self._by_model: dict[str, tuple[CompiledSqlTest, ...]] = { - name: tuple(tests) for name, tests in by_model.items() - } + return {name: tuple(tests) for name, tests in by_model.items()} def all(self) -> tuple[CompiledSqlTest, ...]: return self._tests diff --git a/src/sqlbuild/rule_engine/classes/project_facts.py b/src/sqlbuild/rule_engine/classes/project_facts.py index 89efb1612c..595de795c9 100644 --- a/src/sqlbuild/rule_engine/classes/project_facts.py +++ b/src/sqlbuild/rule_engine/classes/project_facts.py @@ -2,7 +2,13 @@ from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_audit_record import compiled_audit_record +from sqlbuild.compiler.compiled_project.main.compiled_function_record import ( + compiled_function_record, +) +from sqlbuild.compiler.compiled_project.main.compiled_seed_record import compiled_seed_record +from sqlbuild.compiler.compiled_project.main.compiled_source_record import compiled_source_record from sqlbuild.rule_engine.classes.project_tree import ProjectTree, public_model from sqlbuild.rule_engine.models import Model @@ -10,14 +16,16 @@ class ProjectFacts: """Compiler-owned project resources and deterministic project tree.""" - def __init__(self, *, project: CompiledProject, project_dir: Path) -> None: + def __init__(self, *, project: CompiledProjectView, project_dir: Path) -> None: self.models: tuple[Model, ...] = tuple( public_model(model) for model in sorted(project.models, key=lambda item: item.relative_path.as_posix()) ) - self.sources: tuple[object, ...] = project.sources - self.seeds: tuple[object, ...] = project.seeds - self.functions: tuple[object, ...] = project.functions + self.sources: tuple[object, ...] = tuple(map(compiled_source_record, project.sources)) + self.seeds: tuple[object, ...] = tuple(map(compiled_seed_record, project.seeds)) + self.functions: tuple[object, ...] = tuple(map(compiled_function_record, project.functions)) self.tests: tuple[object, ...] = project.sql_tests - self.audits: tuple[object, ...] = project.audits - self.tree: ProjectTree = ProjectTree(project_dir=project_dir, project=project) + self.audits: tuple[object, ...] = tuple(map(compiled_audit_record, project.audits)) + self.tree: ProjectTree = ProjectTree( + project_dir=project_dir, project=project, models=self.models + ) diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index 88f38ae34f..f192e64dae 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -7,7 +7,8 @@ import stat from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledProject +import sqlbuild._native as _native +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.constants import PARENT_DIRECTORY_TOKEN from sqlbuild.rule_engine.exceptions import RuleUsageError from sqlbuild.rule_engine.models import Model, ProjectPath @@ -19,12 +20,21 @@ class ProjectTree: """Deterministic compiler-owned view of supported project paths.""" - def __init__(self, *, project_dir: Path, project: CompiledProject) -> None: + def __init__( + self, + *, + project_dir: Path, + project: CompiledProjectView, + models: tuple[Model, ...] | None = None, + ) -> None: self._root: Path = project_dir.resolve() self._discovered: tuple[ProjectPath, ...] | None = None self._directories: dict[Path, Path | None] = {} + model_views: tuple[Model, ...] = ( + tuple(public_model(item) for item in project.models) if models is None else models + ) self._resource_paths: dict[str, Model] = { - item.relative_path.as_posix(): public_model(item) for item in project.models + item.path.as_posix(): item for item in model_views } @property @@ -144,7 +154,7 @@ def _validate_relative(path: str) -> None: def public_model(model: object) -> Model: from sqlbuild.compiler.compile.models import CompiledModel - if not isinstance(model, CompiledModel): + if not isinstance(model, (CompiledModel, _native.NativeCompiledModel)): raise RuleUsageError("model is not a compiled SQL model") value: object = model.config.values.get("materialized") return Model( diff --git a/src/sqlbuild/rule_engine/classes/rule_context.py b/src/sqlbuild/rule_engine/classes/rule_context.py index f6cbceb13a..0fb4e32544 100644 --- a/src/sqlbuild/rule_engine/classes/rule_context.py +++ b/src/sqlbuild/rule_engine/classes/rule_context.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any, cast -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.audit_facts import AuditFacts from sqlbuild.rule_engine.classes.column_facts import ColumnFacts from sqlbuild.rule_engine.classes.contract_facts import ContractFacts @@ -30,14 +30,15 @@ def build_rule_fact_views( - *, project: CompiledProject, project_dir: Path, dialect: str + *, project: CompiledProjectView, project_dir: Path, dialect: str ) -> RuleFactViews: """Build immutable views once for one compiled project snapshot.""" + project_facts: ProjectFacts = ProjectFacts(project=project, project_dir=project_dir) return RuleFactViews( - project=ProjectFacts(project=project, project_dir=project_dir), + project=project_facts, sql=SqlFacts(dialect=dialect, project=project), - graph=GraphFacts(project=project), + graph=GraphFacts(project=project, models=project_facts.models), columns=ColumnFacts(project=project), contracts=ContractFacts(project=project), tests=TestFacts(project=project), @@ -52,10 +53,10 @@ class EvaluationRuleContext: def __init__( self, *, - model: CompiledModel | None, + model: CompiledModelView | None, rule: Rule, config: RulesConfig, - project: CompiledProject, + project: CompiledProjectView, project_dir: Path, selected_rules: tuple[Rule, ...], dialect: str, diff --git a/src/sqlbuild/rule_engine/classes/sql_facts.py b/src/sqlbuild/rule_engine/classes/sql_facts.py index 3f292d2635..d55c346673 100644 --- a/src/sqlbuild/rule_engine/classes/sql_facts.py +++ b/src/sqlbuild/rule_engine/classes/sql_facts.py @@ -1,6 +1,8 @@ """SQL facts indexed by custom-rule model subjects.""" -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from functools import cached_property + +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.rule_engine.classes.sql_document import SqlDocument from sqlbuild.rule_engine.exceptions import RuleUsageError from sqlbuild.rule_engine.models import Model, ModelSql @@ -9,18 +11,20 @@ class SqlFacts: """Authored and expanded SQL indexed by public model identity.""" - def __init__(self, *, dialect: str, project: CompiledProject) -> None: + def __init__(self, *, dialect: str, project: CompiledProjectView) -> None: self._dialect: str = dialect - self._models: dict[object, CompiledModel] = { - model.relative_path: model for model in project.models - } + self._project: CompiledProjectView = project self._documents: dict[object, ModelSql] = {} + @cached_property + def _models(self) -> dict[object, CompiledModelView]: + return {model.relative_path: model for model in self._project.models} + def for_model(self, model: Model) -> ModelSql: cached: ModelSql | None = self._documents.get(model.path) if cached is not None: return cached - compiled: CompiledModel = self._resolve(model) + compiled: CompiledModelView = self._resolve(model) documents: ModelSql = ModelSql( authored=SqlDocument(source=compiled.authored_sql, dialect=self._dialect), expanded=SqlDocument(source=compiled.query_sql, dialect=self._dialect), @@ -28,7 +32,7 @@ def for_model(self, model: Model) -> ModelSql: self._documents[model.path] = documents return documents - def _resolve(self, model: Model) -> CompiledModel: + def _resolve(self, model: Model) -> CompiledModelView: try: return self._models[model.path] except KeyError as error: diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index f7efc3453e..b9f34c04ad 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -15,7 +15,6 @@ RULES_CACHE_DIRECTORY_NAME: str = "rules-cache" RULES_BULK_CACHE_DIRECTORY_NAME: str = "bulk" CUSTOM_RULES_CACHE_FILE: str = "custom-rules.json" -NATIVE_RULES_MEMO_VERSION: str = "native-rules-response-v2" NATIVE_RULES_MEMO_FILE: str = "native-response.json" SQL_RULES_CACHE_FILE: str = "sql.json" NATIVE_RULES_CACHE_FILE: str = "native.json" diff --git a/src/sqlbuild/rule_engine/main/_evaluate.py b/src/sqlbuild/rule_engine/main/_evaluate.py index f6e6a4036d..b239779aa9 100644 --- a/src/sqlbuild/rule_engine/main/_evaluate.py +++ b/src/sqlbuild/rule_engine/main/_evaluate.py @@ -2,14 +2,14 @@ from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.ruleset import evaluate_project from sqlbuild.rule_engine.models import RulesConfig, RulesResult def evaluate( *, - project: CompiledProject, + project: CompiledProjectView, config: RulesConfig, project_dir: Path, dialect: str = "generic", diff --git a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py b/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py index bac5280adc..d2cb8c0d2e 100644 --- a/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py +++ b/src/sqlbuild/rule_engine/main/_prepare_sql_rules.py @@ -5,13 +5,13 @@ from concurrent.futures import Executor from threading import Event -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.rule_engine._helpers.run.rules import prepare_sql_rules as prepare from sqlbuild.rule_engine.models import PreparedSqlLint def prepare_sql_rules( - *, inputs: CompileProjectInputs, executor: Executor, dialect: str, stop: Event + *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event ) -> PreparedSqlLint | None: """Prepare configured SQL checks without publishing findings before compilation finishes.""" diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index 98831bae24..23af930558 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -425,6 +425,19 @@ class CustomRuleRun: stateful_codes: frozenset[str] = frozenset() +@dataclass(frozen=True) +class NativeRulesEvaluation: + """The built-in rules response, decoded natively.""" + + findings: tuple[Finding, ...] + selected_codes: tuple[str, ...] + evaluated_models: int + cache_hits: int + cache_misses: int + built_in_ms: int + reused: bool + + @dataclass(frozen=True) class CustomRulesOutcome: """Custom-rule finding payloads in canonical order and their cache accounting.""" diff --git a/src/sqlbuild/rule_engine/types.py b/src/sqlbuild/rule_engine/types.py index 477c7e123c..cc82fc5b28 100644 --- a/src/sqlbuild/rule_engine/types.py +++ b/src/sqlbuild/rule_engine/types.py @@ -26,6 +26,7 @@ type RuleOptionValue = bool | int | str | tuple[str, ...] | tuple[int, ...] type FactKey = tuple[str, ...] +type FindingRow = tuple[bool, str, str, int, int, str, str] type CustomRulePlan = dict[str, frozenset[str] | None] type CustomHostPlan = dict[str, list[str] | None] diff --git a/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py b/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py index c7c053acf7..0a96e4bd31 100644 --- a/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py +++ b/src/sqlbuild/spec/contracts/_helpers/dynamic_columns.py @@ -2,8 +2,9 @@ from __future__ import annotations -import re +from functools import lru_cache +import sqlbuild._native as _native from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -12,7 +13,7 @@ def dynamic_family_accepts_name(*, family: SchemaDynamicColumnFamily, column_nam if family.name_pattern is None: return True - return re.fullmatch(family.name_pattern, column_name) is not None + return _compiled_name_pattern(family.name_pattern).matches(column_name) def matching_dynamic_families( @@ -25,3 +26,8 @@ def matching_dynamic_families( for family in families if dynamic_family_accepts_name(family=family, column_name=column_name) ) + + +@lru_cache(maxsize=256) +def _compiled_name_pattern(pattern: str) -> _native.NamePattern: + return _native.NamePattern(pattern) diff --git a/src/sqlbuild/spec/contracts/_helpers/targets.py b/src/sqlbuild/spec/contracts/_helpers/targets.py index 0705dd3d2e..f35d453908 100644 --- a/src/sqlbuild/spec/contracts/_helpers/targets.py +++ b/src/sqlbuild/spec/contracts/_helpers/targets.py @@ -1,22 +1,28 @@ -"""Target configuration resolution implementations.""" +"""Target selection and target merging, resolved natively over the loaded config objects.""" from __future__ import annotations -from dataclasses import replace +from typing import Any +import sqlbuild._native as _native from sqlbuild.spec.contracts.exceptions import SpecConfigError from sqlbuild.spec.contracts.models import ( ClonePolicy, ExecutionLimitsConfig, - LocalClonePolicy, LocalConfig, - LocalTargetConfig, ProjectConfig, TargetConfig, TargetDiffConfig, TargetWarehousesConfig, ) +_GROUPS: dict[str, type[object]] = { + "clone": ClonePolicy, + "execution_limits": ExecutionLimitsConfig, + "diff": TargetDiffConfig, + "warehouses": TargetWarehousesConfig, +} + def resolve_target_name( *, @@ -26,16 +32,12 @@ def resolve_target_name( ) -> str | None: """Resolve the effective target name.""" - target_name: str | None = selected_target - if target_name is None: - target_name = local_config.target - if target_name is None: - target_name = project_config.default_target - if target_name is None: - return None - if target_name not in project_config.targets and target_name not in local_config.targets: - raise SpecConfigError(f"Unknown target '{target_name}'") - return target_name + name, error = _native.effective_config_target_name( + project_config, local_config, selected_target + ) + if error is not None: + raise SpecConfigError(error) + return name def resolve_target_config( @@ -46,200 +48,14 @@ def resolve_target_config( ) -> TargetConfig: """Merge project target config with local developer overrides.""" - return _apply_default_connection( - project_config=project_config, - local_config=local_config, - target_name=target_name, - target_config=_merge_target_config( - project_config=project_config, - local_config=local_config, - target_name=target_name, - ), - ) - - -def _apply_default_connection( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_name: str, - target_config: TargetConfig, -) -> TargetConfig: - """Select the only named connection for a target that does not choose one.""" - - if ( - target_config.connection_name is not None - or target_config.connection - or project_config.connection - or local_config.connection - ): - return target_config - connection_names: list[str] = sorted({*project_config.connections, *local_config.connections}) - if not connection_names: - return target_config - if len(connection_names) > 1: - names: str = ", ".join(connection_names) - raise SpecConfigError( - f"targets.{target_name} does not set connection and several named connections " - f'are defined ({names}); set connection = "" on [targets.{target_name}] in ' - "sqlbuild_project.toml or sqlbuild_local.toml" - ) - return replace(target_config, connection_name=connection_names[0]) - - -def _merge_target_config( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_name: str, -) -> TargetConfig: - project_target: TargetConfig = project_config.targets.get(target_name, TargetConfig()) - local_target: LocalTargetConfig | None = local_config.targets.get(target_name) - if local_target is None: - return project_target - local_overrides_retention: bool = ( - local_target.time_travel_retention is not None - or local_target.time_travel_retention_by_materialization is not None - ) - target_config: TargetConfig = TargetConfig( - connection={**project_target.connection, **local_target.connection}, - connection_name=( - local_target.connection_name - if local_target.connection_name is not None - else project_target.connection_name - ), - vars={**project_target.vars, **local_target.vars}, - database=( - local_target.database if local_target.database is not None else project_target.database - ), - schema=local_target.schema if local_target.schema is not None else project_target.schema, - loader_schema=( - local_target.loader_schema - if local_target.loader_schema is not None - else project_target.loader_schema - ), - defer_sources_to=( - local_target.defer_sources_to - if local_target.defer_sources_to is not None - else project_target.defer_sources_to - ), - defer_clone_from=( - local_target.defer_clone_from - if local_target.defer_clone_from is not None - else project_target.defer_clone_from - ), - compile_cache=( - local_target.compile_cache - if local_target.compile_cache is not None - else project_target.compile_cache - ), - time_travel_retention=( - local_target.time_travel_retention - if local_overrides_retention - else project_target.time_travel_retention - ), - time_travel_retention_by_materialization=( - local_target.time_travel_retention_by_materialization or {} - if local_overrides_retention - else project_target.time_travel_retention_by_materialization - ), - owns_time_travel_retention_namespace=( - local_target.owns_time_travel_retention_namespace - if local_target.owns_time_travel_retention_namespace is not None - else project_target.owns_time_travel_retention_namespace - ), - default_table_type=( - local_target.default_table_type - if local_target.default_table_type is not None - else project_target.default_table_type - ), - table_type_downgrade=( - local_target.table_type_downgrade - if local_target.table_type_downgrade is not None - else project_target.table_type_downgrade - ), - time_travel_retention_decrease=( - local_target.time_travel_retention_decrease - if local_target.time_travel_retention_decrease is not None - else project_target.time_travel_retention_decrease - ), - missing_migration_origin=( - local_target.missing_migration_origin - if local_target.missing_migration_origin is not None - else project_target.missing_migration_origin - ), - execution_limits=_merge_execution_limits( - project_limits=project_target.execution_limits, - local_limits=local_target.execution_limits, - ), - clone=_merge_clone_policy( - project_clone=project_target.clone, - local_clone=local_target.clone, - ), - diff=TargetDiffConfig( - max_full_rows=( - local_target.diff.max_full_rows - if local_target.diff.max_full_rows is not None - else project_target.diff.max_full_rows - ) - ), - warehouses=TargetWarehousesConfig( - build=( - local_target.warehouses.build - if local_target.warehouses.build is not None - else project_target.warehouses.build - ), - query=( - local_target.warehouses.query - if local_target.warehouses.query is not None - else project_target.warehouses.query - ), - ), - ) - return target_config - - -def _merge_execution_limits( - *, project_limits: ExecutionLimitsConfig, local_limits: ExecutionLimitsConfig -) -> ExecutionLimitsConfig: - return ExecutionLimitsConfig( - max_models=( - local_limits.max_models - if local_limits.max_models is not None - else project_limits.max_models - ), - max_duration=( - local_limits.max_duration - if local_limits.max_duration is not None - else project_limits.max_duration - ), - max_duration_seconds=( - local_limits.max_duration_seconds - if local_limits.max_duration is not None - else project_limits.max_duration_seconds - ), - remediation=( - local_limits.remediation - if local_limits.remediation is not None - else project_limits.remediation - ), - ) - - -def _merge_clone_policy( - *, project_clone: ClonePolicy, local_clone: LocalClonePolicy -) -> ClonePolicy: - allow_as_clone_origin: bool | None = local_clone.allow_as_clone_origin - allow_as_clone_destination: bool | None = local_clone.allow_as_clone_destination - return ClonePolicy( - allow_as_clone_origin=( - allow_as_clone_origin - if allow_as_clone_origin is not None - else project_clone.allow_as_clone_origin - ), - allow_as_clone_destination=( - allow_as_clone_destination - if allow_as_clone_destination is not None - else project_clone.allow_as_clone_destination - ), + rows, error = _native.effective_config_target_fields( + (project_config, local_config), target_name, TargetConfig() ) + if rows is None: + raise SpecConfigError(error or "") + values: dict[str, Any] = {} + grouped: dict[str, dict[str, Any]] = {group: {} for group in _GROUPS} + for group, name, value in rows: + (values if group is None else grouped[group])[name] = value + nested: dict[str, Any] = {group: kind(**grouped[group]) for group, kind in _GROUPS.items()} + return TargetConfig(**values, **nested) diff --git a/src/sqlbuild/sql_values/_helpers/normalization.py b/src/sqlbuild/sql_values/_helpers/normalization.py deleted file mode 100644 index a2cbdf7948..0000000000 --- a/src/sqlbuild/sql_values/_helpers/normalization.py +++ /dev/null @@ -1,296 +0,0 @@ -"""Recursive implementation for typed SQL value normalization.""" - -from __future__ import annotations - -import math -from dataclasses import dataclass -from decimal import Decimal, InvalidOperation -from typing import cast - -from sqlbuild.sql_values.constants import ( - MAX_SIGNED_64_BIT_INTEGER, - MIN_SIGNED_64_BIT_INTEGER, -) -from sqlbuild.sql_values.exceptions import SqlValueValidationError -from sqlbuild.sql_values.models import AuthoredSqlSet, SqlLogicalType, SqlValue, SqlValueLimits -from sqlbuild.sql_values.types import SqlScalar, SqlValueKind - -_SCALAR_KINDS: frozenset[SqlValueKind] = frozenset( - { - SqlValueKind.STRING, - SqlValueKind.INTEGER, - SqlValueKind.BOOLEAN, - SqlValueKind.FLOAT, - SqlValueKind.DECIMAL, - SqlValueKind.NULL, - } -) -_DEFAULT_SQL_VALUE_LIMITS: SqlValueLimits = SqlValueLimits() - - -@dataclass -class _NormalizationState: - context: str - limits: SqlValueLimits - elements: int = 0 - size: int = 0 - - def account(self, *, path: str, elements: int = 0, size: int = 0) -> None: - self.elements += elements - self.size += size - if self.elements > self.limits.max_elements: - raise SqlValueValidationError( - f"{self.context}{path} exceeds the maximum element count " - f"of {self.limits.max_elements}" - ) - if self.size > self.limits.max_size: - raise SqlValueValidationError( - f"{self.context}{path} exceeds the maximum value size of " - f"{self.limits.max_size} bytes" - ) - - -def normalize_sql_value( - *, - raw_value: object, - context: str, - explicit_type: str | None = None, - limits: SqlValueLimits = _DEFAULT_SQL_VALUE_LIMITS, -) -> SqlValue: - """Validate and normalize one authored value with contextual diagnostics.""" - - state: _NormalizationState = _NormalizationState(context=context, limits=limits) - return _normalize( - raw_value=raw_value, - path="", - depth=1, - state=state, - explicit_type=explicit_type, - ) - - -def sql_value_identity(*, value: SqlValue) -> tuple[object, ...]: - """Return typed canonical identity without Python's cross-type equality.""" - - kind: SqlValueKind = value.kind - if kind == SqlValueKind.DECIMAL: - decimal_value: object = value.value - if not isinstance(decimal_value, Decimal): - raise SqlValueValidationError("typed decimal contains an invalid payload") - return kind.value, decimal_value.normalize().as_tuple() - if kind in _SCALAR_KINDS: - return kind.value, value.value - if kind in {SqlValueKind.LIST, SqlValueKind.SET}: - items: tuple[SqlValue, ...] = cast(tuple[SqlValue, ...], value.value) - return kind.value, tuple(sql_value_identity(value=item) for item in items) - entries: tuple[tuple[str, SqlValue], ...] = cast(tuple[tuple[str, SqlValue], ...], value.value) - return kind.value, tuple((key, sql_value_identity(value=item)) for key, item in entries) - - -def _normalize( - *, - raw_value: object, - path: str, - depth: int, - state: _NormalizationState, - explicit_type: str | None, -) -> SqlValue: - if depth > state.limits.max_depth: - raise SqlValueValidationError( - f"{state.context}{path} exceeds the maximum nesting depth of {state.limits.max_depth}" - ) - if explicit_type is not None: - return _normalize_explicit_scalar( - raw_value=raw_value, path=path, explicit_type=explicit_type, state=state - ) - if raw_value is None: - state.account(path=path, size=4) - return _scalar(kind=SqlValueKind.NULL, value=None) - if type(raw_value) is bool: - state.account(path=path, size=5) - return _scalar(kind=SqlValueKind.BOOLEAN, value=raw_value) - if type(raw_value) is int: - if not MIN_SIGNED_64_BIT_INTEGER <= raw_value <= MAX_SIGNED_64_BIT_INTEGER: - raise SqlValueValidationError( - f"{state.context}{path} integer {raw_value} is outside the signed 64-bit range" - ) - state.account(path=path, size=len(str(raw_value))) - return _scalar(kind=SqlValueKind.INTEGER, value=raw_value) - if type(raw_value) is float: - if not math.isfinite(raw_value): - raise SqlValueValidationError( - f"{state.context}{path} float must be finite; received {raw_value!r}" - ) - normalized_float: float = 0.0 if raw_value == 0.0 else raw_value - state.account(path=path, size=len(repr(normalized_float))) - return _scalar(kind=SqlValueKind.FLOAT, value=normalized_float) - if type(raw_value) is Decimal: - if not raw_value.is_finite(): - raise SqlValueValidationError( - f"{state.context}{path} decimal must be finite; received {raw_value!r}" - ) - state.account(path=path, size=len(str(raw_value))) - return _scalar(kind=SqlValueKind.DECIMAL, value=raw_value) - if type(raw_value) is str: - state.account(path=path, size=len(raw_value.encode("utf-8")) + 2) - return _scalar(kind=SqlValueKind.STRING, value=raw_value) - if type(raw_value) is list: - return _normalize_collection( - raw_values=tuple(raw_value), kind=SqlValueKind.LIST, path=path, depth=depth, state=state - ) - if isinstance(raw_value, AuthoredSqlSet): - return _normalize_collection( - raw_values=raw_value.values, - kind=SqlValueKind.SET, - path=path, - depth=depth, - state=state, - ) - if type(raw_value) is dict: - return _normalize_object( - raw_value=cast(dict[object, object], raw_value), path=path, depth=depth, state=state - ) - raise SqlValueValidationError( - f"{state.context}{path} has unsupported type {type(raw_value).__name__}; " - "expected a scalar, list, set, or object" - ) - - -def _normalize_explicit_scalar( - *, raw_value: object, path: str, explicit_type: str, state: _NormalizationState -) -> SqlValue: - try: - kind: SqlValueKind = SqlValueKind(explicit_type.lower()) - except ValueError as error: - allowed: str = ", ".join(item.value for item in SqlValueKind if item in _SCALAR_KINDS) - raise SqlValueValidationError( - f"{state.context}{path} has unsupported explicit type '{explicit_type}'; " - f"expected {allowed}" - ) from error - if kind not in _SCALAR_KINDS: - raise SqlValueValidationError( - f"{state.context}{path} explicit type '{explicit_type}' is not a scalar type" - ) - if kind == SqlValueKind.DECIMAL: - if type(raw_value) is not str: - raise SqlValueValidationError( - f"{state.context}{path} with type decimal requires a quoted decimal string; " - f"received {type(raw_value).__name__}" - ) - try: - decimal_value: Decimal = Decimal(raw_value) - except InvalidOperation as error: - raise SqlValueValidationError( - f"{state.context}{path} has invalid decimal value {raw_value!r}" - ) from error - if not decimal_value.is_finite(): - raise SqlValueValidationError( - f"{state.context}{path} decimal must be finite; received {raw_value!r}" - ) - state.account(path=path, size=len(raw_value)) - return _scalar(kind=kind, value=decimal_value) - expected_types: dict[SqlValueKind, type[object]] = { - SqlValueKind.STRING: str, - SqlValueKind.INTEGER: int, - SqlValueKind.BOOLEAN: bool, - SqlValueKind.FLOAT: float, - SqlValueKind.NULL: type(None), - } - expected_type: type[object] = expected_types[kind] - valid: bool = type(raw_value) is expected_type - if not valid: - raise SqlValueValidationError( - f"{state.context}{path} has type {type(raw_value).__name__}; expected {kind.value}" - ) - return _normalize(raw_value=raw_value, path=path, depth=1, state=state, explicit_type=None) - - -def _normalize_collection( - *, - raw_values: tuple[object, ...], - kind: SqlValueKind, - path: str, - depth: int, - state: _NormalizationState, -) -> SqlValue: - if not raw_values: - raise SqlValueValidationError( - f"{state.context}{path} {kind.value} must contain at least one value" - ) - state.account(path=path, elements=len(raw_values), size=2 + len(raw_values)) - values: tuple[SqlValue, ...] = tuple( - _normalize( - raw_value=raw_value, - path=f"{path}[{index}]", - depth=depth + 1, - state=state, - explicit_type=None, - ) - for index, raw_value in enumerate(raw_values) - ) - element_type: SqlLogicalType = _infer_element_type( - values=values, path=path, kind=kind, state=state - ) - if kind == SqlValueKind.SET: - seen: set[tuple[object, ...]] = set() - for index, value in enumerate(values): - identity: tuple[object, ...] = sql_value_identity(value=value) - if identity in seen: - raise SqlValueValidationError( - f"{state.context}{path} contains duplicate set value {value.value!r} " - f"at {path}[{index}]" - ) - seen.add(identity) - values = tuple(sorted(values, key=lambda value: repr(sql_value_identity(value=value)))) - return SqlValue(logical_type=SqlLogicalType(kind, element_type), value=values) - - -def _infer_element_type( - *, values: tuple[SqlValue, ...], path: str, kind: SqlValueKind, state: _NormalizationState -) -> SqlLogicalType: - expected: SqlLogicalType | None = next( - (value.logical_type for value in values if value.kind != SqlValueKind.NULL), None - ) - if expected is None: - raise SqlValueValidationError( - f"{state.context}{path} {kind.value} cannot infer an element type from only null values" - ) - for index, value in enumerate(values): - if value.kind != SqlValueKind.NULL and value.logical_type != expected: - raise SqlValueValidationError( - f"{state.context}{path}[{index}] has type {value.logical_type.display_name}; " - f"expected {expected.display_name}" - ) - return expected - - -def _normalize_object( - *, raw_value: dict[object, object], path: str, depth: int, state: _NormalizationState -) -> SqlValue: - state.account(path=path, elements=len(raw_value), size=2 + len(raw_value)) - entries: list[tuple[str, SqlValue]] = [] - for raw_key, item in raw_value.items(): - if type(raw_key) is not str: - raise SqlValueValidationError( - f"{state.context}{path} object key {raw_key!r} must be a string" - ) - state.account(path=path, size=len(raw_key.encode("utf-8")) + 2) - item_path: str = f"{path}.{raw_key}" if path else f".{raw_key}" - entries.append( - ( - raw_key, - _normalize( - raw_value=item, - path=item_path, - depth=depth + 1, - state=state, - explicit_type=None, - ), - ) - ) - entries.sort(key=lambda entry: entry[0]) - return SqlValue(logical_type=SqlLogicalType(SqlValueKind.OBJECT), value=tuple(entries)) - - -def _scalar(*, kind: SqlValueKind, value: SqlScalar) -> SqlValue: - return SqlValue(logical_type=SqlLogicalType(kind), value=value) diff --git a/src/sqlbuild/sql_values/main/normalize.py b/src/sqlbuild/sql_values/main/normalize.py index 40627c5f61..bb375ae989 100644 --- a/src/sqlbuild/sql_values/main/normalize.py +++ b/src/sqlbuild/sql_values/main/normalize.py @@ -1,6 +1,8 @@ """Public entry point for typed SQL value normalization.""" -from sqlbuild.sql_values._helpers.normalization import normalize_sql_value as _normalize_sql_value +from typing import cast + +import sqlbuild._native as _native from sqlbuild.sql_values.models import SqlValue, SqlValueLimits @@ -13,10 +15,12 @@ def normalize_sql_value( ) -> SqlValue: """Validate and normalize one authored value with contextual diagnostics.""" - if limits is None: - return _normalize_sql_value( - raw_value=raw_value, context=context, explicit_type=explicit_type - ) - return _normalize_sql_value( - raw_value=raw_value, context=context, explicit_type=explicit_type, limits=limits + applied: SqlValueLimits = SqlValueLimits() if limits is None else limits + return cast( + SqlValue, + _native.normalize_typed_sql_value( + raw_value, + (context, explicit_type), + (applied.max_depth, applied.max_elements, applied.max_size), + ), ) diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py index fd86575f11..0c7b0eda7a 100644 --- a/tests/e2e/scripts/compiler_differential/_test_types.py +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -44,7 +44,7 @@ class FailureCorpusCodesTestCase: """The failure corpus run for real and the codes its cases must emit.""" description: str - expected_first_errors: dict[str, str] + expected_first_errors: dict[str, str | None] expected_first_messages: dict[str, str] expected_warnings: dict[str, str] expected_discovery_codes: frozenset[str] diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index 117529835c..a611640681 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -12,6 +12,9 @@ import dataclasses import sys +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record + import sqlbuild.compiler.frontier.main._compile_frontier as frontier from sqlbuild.compiler.frontier.types import CompilerStage @@ -22,11 +25,12 @@ def _perturbed(*, until, **stages): result = _original(until=until, **stages) if until is not CompilerStage.COMPILED_PROJECT or not result.models: return result - first = result.models[0] + result = compiled_project_record(result) + first = compiled_model_record(result.models[0]) changed = dataclasses.replace( first, query_sql=f"SELECT * FROM ({{first.query_sql}}) AS perturbed_by_test" ) - return dataclasses.replace(result, models=(changed, *result.models[1:])) + return dataclasses.replace(result, models=(changed, *result.models[1:]), native_project=None) frontier.native_frontier = _perturbed @@ -55,6 +59,9 @@ def _perturbed(*, until, **stages): RENDER_PERTURBATION: str = """ import dataclasses +from sqlbuild.compiler.compile.models import CompileModelInput +from sqlbuild.compiler.compiled_project.main.inputs_with import inputs_with + import sqlbuild.compiler.frontier.main._compile_frontier as frontier from sqlbuild.compiler.frontier.types import CompilerStage @@ -65,9 +72,12 @@ def _perturbed(*, until, **stages): result = _original(until=until, **stages) if until is not CompilerStage.COMPILE_PROJECT_INPUTS or not result.model_inputs: return result - first = result.model_inputs[0] + first = CompileModelInput(**{ + field.name: getattr(result.model_inputs[0], field.name) + for field in dataclasses.fields(CompileModelInput) + }) changed = dataclasses.replace(first, macro_deps=(*first.macro_deps, "perturbed_by_test")) - return dataclasses.replace(result, model_inputs=(changed, *result.model_inputs[1:])) + return inputs_with(inputs=result, model_inputs=(changed, *result.model_inputs[1:])) frontier.native_frontier = _perturbed diff --git a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py index cb3524f5f4..a6723a4de6 100644 --- a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py +++ b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py @@ -16,6 +16,7 @@ emitted_failure_codes, render_error_codes, ) +from scripts.compiler_differential._helpers.refactor_corpus.refactor_cases import refactor_cases from scripts.compiler_differential.constants import ( ANALYSIS_UNREACHABLE_CODES, DISCOVERY_UNREACHABLE_CODES, @@ -34,7 +35,8 @@ FailureCorpusCodesTestCase( description="native_engine", expected_first_errors={ - f"failure/{case.name}": case.expected_code for case in failure_cases() + **{f"failure/{case.name}": case.expected_code for case in failure_cases()}, + **{project.name: project.expected.error_code for project in refactor_cases()}, }, expected_first_messages={ f"failure/{case.name}": case.expected_message diff --git a/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py b/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py index ad31837751..0064e8b9d1 100644 --- a/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py +++ b/tests/e2e/scripts/compiler_differential/test_native_fallback_gate.py @@ -37,8 +37,8 @@ expected_exit_code=1, expected_lines=( "Native fallback allow-list: native-preview macro_calls " - "macro_calls.resolution deferred (project): listed but no longer occurs; remove it " - "from the allow-list", + "macro_calls.resolution deferred (project): fallback disappeared but native " + "answers did not appear; the stage may be switched off.", ), ), NativeFallbackGateTestCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 45958b6c3a..0431c94438 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -876,8 +876,18 @@ class SnapshotReappearingKeyBuildE2ETestCase: @dataclass(frozen=True) class DynamicColumnContractE2ETestCase: description: str - project_name: str expected_rows: tuple[tuple[object, ...], ...] + name_pattern_entry: str = "" + + +@dataclass(frozen=True) +class DynamicColumnRejectionE2ETestCase: + """Test case for a dynamic column name pattern that fails a build.""" + + description: str + name_pattern_entry: str + expected_output: str + expected_returncode: int = 1 @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 514ba07286..c3856bf578 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1056,3 +1056,62 @@ def coded_json_diagnostic(*, document: dict[str, Any], code: str) -> dict[str, A """Return the first compile diagnostic with the given code.""" return {item["code"]: item for item in document["diagnostics"]}[code] + + +def dynamic_category_project(*, tmp_path: Path, name_pattern_entry: str) -> Path: + """Create a project pivoting order amounts into a governed dynamic column family.""" + + return prepare_inline_project( + tmp_path=tmp_path, + project_name="dynamic_category_amounts", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "dynamic_category_amounts"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "warehouse.duckdb"\n\n' + "[defaults]\n" + 'contract = "enforced"\n\n' + "[rules]\n" + 'select = ["SQBRMODEL102", "SQBRCONTRACT101", "SQBRCONTRACT105", ' + '"SQBRCONTRACT106"]\n' + ), + "models/stg_order_amounts.sql": ( + "MODEL (description 'Test model stg_order_amounts.',\n" + " materialized table,\n" + " columns (\n" + " customer_id (type INTEGER),\n" + " category (type VARCHAR),\n" + ' amount (type "DECIMAL(12,2)"),\n' + " ),\n" + ");\n" + "SELECT CAST(1 AS INTEGER) AS customer_id, " + "CAST('books' AS VARCHAR) AS category, " + "CAST(10.25 AS DECIMAL(12,2)) AS amount\n" + "UNION ALL\n" + "SELECT CAST(1 AS INTEGER), CAST('games' AS VARCHAR), " + "CAST(20.50 AS DECIMAL(12,2))\n" + "UNION ALL\n" + "SELECT CAST(2 AS INTEGER), CAST('books' AS VARCHAR), " + "CAST(7.00 AS DECIMAL(12,2))\n" + ), + "models/customer_category_amounts.sql": ( + "MODEL (description 'Test model customer_category_amounts.',\n" + " materialized table,\n" + " columns (customer_id (type INTEGER)),\n" + " dynamic_columns (\n" + " category_amounts (\n" + " pivot_column category,\n" + " value_column amount,\n" + " aggregate MAX,\n" + ' type "DECIMAL(12,2)"' + name_pattern_entry + "\n" + " )\n" + " ),\n" + ");\n" + 'PIVOT __ref("stg_order_amounts")\n' + "ON category\n" + "USING MAX(amount)\n" + "GROUP BY customer_id\n" + ), + }, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py index aa73f3aca0..0176e83d89 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dynamic_column_contract.py @@ -9,12 +9,10 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( DynamicColumnContractE2ETestCase, + DynamicColumnRejectionE2ETestCase, ) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - query_duckdb, - run_sqb, -) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import dynamic_category_project +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb, run_sqb @pytest.mark.parametrize( @@ -22,9 +20,13 @@ ( DynamicColumnContractE2ETestCase( description="generated category columns are enforced", - project_name="dynamic_category_amounts", expected_rows=((1, "10.25", "20.50"), (2, "7.00", None)), ), + DynamicColumnContractE2ETestCase( + description="a name pattern must match each generated column name whole", + expected_rows=((1, "10.25", "20.50"), (2, "7.00", None)), + name_pattern_entry=",\n name_pattern 'books|games'", + ), ), ids=lambda case: case.description, ) @@ -32,65 +34,15 @@ def test_given_dynamic_pivot_contract_when_building_then_generated_columns_are_t test_case: DynamicColumnContractE2ETestCase, tmp_path: Path, ) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=test_case.project_name, - repo_files={ - "sqlbuild_project.toml": ( - 'name = "dynamic_category_amounts"\n' - 'adapter = "duckdb"\n\n' - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[defaults]\n" - 'contract = "enforced"\n\n' - "[rules]\n" - 'select = ["SQBRMODEL102", "SQBRCONTRACT101", "SQBRCONTRACT105", ' - '"SQBRCONTRACT106"]\n' - ), - "models/stg_order_amounts.sql": ( - "MODEL (description 'Test model stg_order_amounts.',\n" - " materialized table,\n" - " columns (\n" - " customer_id (type INTEGER),\n" - " category (type VARCHAR),\n" - ' amount (type "DECIMAL(12,2)"),\n' - " ),\n" - ");\n" - "SELECT CAST(1 AS INTEGER) AS customer_id, " - "CAST('books' AS VARCHAR) AS category, " - "CAST(10.25 AS DECIMAL(12,2)) AS amount\n" - "UNION ALL\n" - "SELECT CAST(1 AS INTEGER), CAST('games' AS VARCHAR), " - "CAST(20.50 AS DECIMAL(12,2))\n" - "UNION ALL\n" - "SELECT CAST(2 AS INTEGER), CAST('books' AS VARCHAR), " - "CAST(7.00 AS DECIMAL(12,2))\n" - ), - "models/customer_category_amounts.sql": ( - "MODEL (description 'Test model customer_category_amounts.',\n" - " materialized table,\n" - " columns (customer_id (type INTEGER)),\n" - " dynamic_columns (\n" - " category_amounts (\n" - " pivot_column category,\n" - " value_column amount,\n" - " aggregate MAX,\n" - ' type "DECIMAL(12,2)"\n' - " )\n" - " ),\n" - ");\n" - 'PIVOT __ref("stg_order_amounts")\n' - "ON category\n" - "USING MAX(amount)\n" - "GROUP BY customer_id\n" - ), - }, + project_dir: Path = dynamic_category_project( + tmp_path=tmp_path, name_pattern_entry=test_case.name_pattern_entry ) result: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build"), project_dir=project_dir, ) + assert result.returncode == 0, result.stdout + result.stderr rows: list[tuple[object, ...]] = query_duckdb( db_path=project_dir / "warehouse.duckdb", @@ -99,5 +51,38 @@ def test_given_dynamic_pivot_contract_when_building_then_generated_columns_are_t "FROM main.customer_category_amounts ORDER BY customer_id" ), ) - assert tuple(rows) == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + ( + DynamicColumnRejectionE2ETestCase( + description="a pattern matching only part of a generated column name rejects it", + name_pattern_entry=",\n name_pattern 'book'", + expected_output="runtime contract has extra columns: books, games", + ), + DynamicColumnRejectionE2ETestCase( + description="an unsupported look-around name pattern is a compile error", + name_pattern_entry=",\n name_pattern '(?=books)books'", + expected_output="has invalid name_pattern: look-around", + ), + ), + ids=lambda case: case.description, +) +def test_given_dynamic_name_pattern_when_building_then_mismatches_and_unsupported_patterns_fail( + test_case: DynamicColumnRejectionE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = dynamic_category_project( + tmp_path=tmp_path, name_pattern_entry=test_case.name_pattern_entry + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), + project_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_returncode, output + assert test_case.expected_output in output, output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py index c542bcdfe9..49fe59ce12 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py @@ -33,7 +33,7 @@ "target/run/functions/python/is_completed_order_py.sql", ), expected_exit_code=0, - expected_compiler_cache_count=2, + expected_compiler_cache_count=1, expected_local_history=False, expected_compiled_paths=( "target/compiled/models/marts/fact_orders.sql", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py index 76762a2cc5..1eeae7d7a0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/_test_types.py @@ -12,3 +12,27 @@ class NativeFastLineageCliTestCase: lineage_targets: tuple[str, ...] expected_lineage: dict[str, dict[str, object]] expected_minimum_traced_edges: int + + +@dataclass(frozen=True) +class NativeRichLineageFailureCliTestCase: + """A native rich lineage failure injected at the binding, compiled through the CLI.""" + + description: str + files: dict[str, str] + command: tuple[str, ...] + expected_returncode: int + expected_error_line: str + expected_wheel_calls: int + + +@dataclass(frozen=True) +class NativeRichLineageCliTestCase: + """Every engine compiling and tracing rich lineage of one project through the CLI.""" + + description: str + files: dict[str, str] + lineage_targets: tuple[str, ...] + expected_lineage: dict[str, dict[str, object]] + expected_wheel_calls: int + expected_minimum_traced_edges: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py index fab6ff3cec..d25f140a6c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/helpers.py @@ -3,6 +3,7 @@ import json import re import subprocess +from collections import Counter from itertools import product from pathlib import Path from typing import NamedTuple, cast @@ -13,20 +14,31 @@ _TIMING: re.Pattern[str] = re.compile(r" \(\d+\.\d+s\)") _DIRECTIONS: tuple[str, ...] = ("upstream", "downstream") +FALLBACK_SITE: str = ( + "compiler/lineage/_helpers/fast_columns.py:_build_polyglot_fast_model_column_lineage" +) class EngineLineageRun(NamedTuple): - """One engine's compile report and lineage traces.""" + """One engine's compile report, lineage traces, wheel calls and deferrals.""" compile_report: str compile_returncode: int traces: list[tuple[int, str, str]] + fallback_parses: int + lineage_deferrals: int + wheel_calls: int def engine_lineage_run( - *, project_dir: Path, files: dict[str, str], engine: str, targets: tuple[str, ...] + *, + project_dir: Path, + files: dict[str, str], + engine: str, + targets: tuple[str, ...], + mode: str = "fast", ) -> EngineLineageRun: - """Write the project, then compile and trace every target both ways under one engine.""" + """Write the project, then compile and trace every target both ways in one lineage mode.""" for relative_path, contents in files.items(): path: Path = project_dir / "project" / relative_path @@ -40,7 +52,7 @@ def engine_lineage_run( engine_args: tuple[str, ...] = ("--compiler-engine", engine) compiled: subprocess.CompletedProcess[str] = run_installed_sqb( project_dir=project_dir / "project", - args=(*engine_args, "compile", "--json", "--no-cache"), + args=(*engine_args, "compile", "--json", "--no-cache", "--lineage-mode", mode), env=environment, ) report: dict[str, object] = json.loads(compiled.stdout.replace(str(project_dir), "")) @@ -55,7 +67,7 @@ def engine_lineage_run( "lineage", target, "--mode", - "fast", + mode, "--direction", direction, "--json", @@ -64,10 +76,55 @@ def engine_lineage_run( ) stdout: str = result.stdout.replace(str(project_dir), "") traces.append((result.returncode, stdout, _TIMING.sub("", result.stderr))) + wheel_calls: Counter[str] = _site_calls(record_dir) return EngineLineageRun( compile_report=json.dumps(report, indent=2), compile_returncode=compiled.returncode, traces=traces, + fallback_parses=wheel_calls[FALLBACK_SITE], + lineage_deferrals=sum( + path.read_text("utf-8").count('"site": "fast_columns.py"') + for path in record_dir.glob("analysis-deferrals-*.jsonl") + ), + wheel_calls=wheel_calls.total(), + ) + + +class FailedLineageRun(NamedTuple): + """A failed command's return code, last stderr line and polyglot wheel calls.""" + + returncode: int + last_error_line: str + wheel_calls: int + + +def failed_rich_lineage_run( + *, project_dir: Path, files: dict[str, str], args: tuple[str, ...], perturbation: str +) -> FailedLineageRun: + """Run one command with `perturbation` installed as `sitecustomize` in the CLI process.""" + + for relative_path, contents in files.items(): + path: Path = project_dir / "project" / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + extlib: Path = project_dir / "extlib" + extlib.mkdir() + _ = (extlib / "sitecustomize.py").write_text(perturbation, encoding="utf-8") + record_dir: Path = project_dir / "records" + result: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir / "project", + args=args, + env={ + REUSE_DISABLE_ENV_VAR: "1", + ANALYSIS_RECORD_DIR_ENV_VAR: str(record_dir), + "PYTHONPATH": str(extlib), + "PYTHONDONTWRITEBYTECODE": "1", + }, + ) + return FailedLineageRun( + returncode=result.returncode, + last_error_line=(result.stderr.strip().splitlines() or [""])[-1], + wheel_calls=_site_calls(record_dir).total(), ) @@ -79,3 +136,11 @@ def model_lineage_summaries(compile_report: str) -> dict[str, object]: list[dict[str, object]], cast(dict[str, object], payload["resources"])["models"] ) return {str(model["name"]): model["lineage"] for model in models} + + +def _site_calls(record_dir: Path) -> Counter[str]: + calls: Counter[str] = Counter() + for path in record_dir.glob("polyglot-sites-*.json"): + for site, _, count in json.loads(path.read_text("utf-8"))["calls"]: + calls[site] += count + return calls diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py index f1a07bf755..96394cfc93 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py @@ -8,10 +8,14 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.fast_lineage._test_types import ( NativeFastLineageCliTestCase, + NativeRichLineageCliTestCase, + NativeRichLineageFailureCliTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.fast_lineage.helpers import ( EngineLineageRun, + FailedLineageRun, engine_lineage_run, + failed_rich_lineage_run, model_lineage_summaries, ) @@ -67,6 +71,63 @@ } +_RICH_FILES: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT_FILES["sqlbuild_project.toml"], + "sources/raw.yml": _PROJECT_FILES["sources/raw.yml"], + "seeds/regions.csv": "region_id,label\n1,north\n", + "seeds/regions.yml": ( + "seeds:\n - name: regions\n description: Regions.\n columns:\n" + " - name: region_id\n type: INTEGER\n - name: label\n type: VARCHAR\n" + ), + "models/staging/stg_orders.sql": _PROJECT_FILES["models/staging/stg_orders.sql"], + "models/marts/order_facts.sql": ( + 'MODEL (description "Order facts");\n\n' + "WITH ranked AS (\n" + " SELECT o.*, r.label, CAST(o.amount AS BIGINT) AS whole_amount\n" + ' FROM __ref("stg_orders") AS o\n' + ' LEFT JOIN __seed("regions") AS r ON o.customer_id = r.region_id\n' + ")\n" + "SELECT order_id, customer_id, COALESCE(label, 'none') AS region_label,\n" + " whole_amount, amount * 2 AS doubled, 'fixed' AS tag\n" + "FROM ranked\n" + ), + "models/marts/customer_totals.sql": ( + 'MODEL (description "Totals per customer");\n\n' + "SELECT customer_id, SUM(doubled) AS total_doubled, COUNT(*) AS order_count,\n" + " MAX(region_label) AS region_label\n" + 'FROM __ref("order_facts")\nGROUP BY customer_id\n' + "UNION ALL\n" + 'SELECT customer_id, amount, 1, NULL FROM __source("raw_orders")\n' + ), + "models/marts/customer_copies.sql": ( + 'MODEL (description "Customer copies");\n\n' + 'SELECT t.*, f.tag FROM __ref("customer_totals") AS t\n' + 'JOIN __ref("order_facts") AS f USING (customer_id)\n' + ), +} + +_NATIVE_PANIC_ERROR: str = "NativeCompilerError: native SQL compilation panicked" +_RICH_PANIC_INJECTION: str = f""" +import sqlbuild._native as native + + +def _panicked(request): + raise native.NativeCompilerError({_NATIVE_PANIC_ERROR!r}) + + +native.build_rich_column_lineage = _panicked +""" +_RICH_COMPILE: tuple[str, ...] = ( + "--compiler-engine", + "native", + "compile", + "--json", + "--no-cache", + "--lineage-mode", + "rich", +) + + @pytest.mark.parametrize( "test_case", [ @@ -137,5 +198,120 @@ def test_given_project_when_compiling_with_each_engine_then_fast_lineage_output_ assert runs["native-preview"] == native +@pytest.mark.parametrize( + "test_case", + [ + NativeRichLineageCliTestCase( + description="stars, CTEs, joins, seeds, set operations and aggregates", + files=_RICH_FILES, + lineage_targets=( + "customer_copies.total_doubled", + "order_facts.whole_amount", + "stg_orders.amount", + ), + expected_lineage={ + "customer_copies": { + "available": True, + "column_count": 5, + "edge_count": 8, + "has_star": True, + }, + "customer_totals": { + "available": True, + "column_count": 4, + "edge_count": 5, + "has_star": False, + }, + "order_facts": { + "available": True, + "column_count": 6, + "edge_count": 5, + "has_star": False, + }, + "stg_orders": { + "available": True, + "column_count": 3, + "edge_count": 3, + "has_star": True, + }, + }, + expected_wheel_calls=0, + expected_minimum_traced_edges=4, + ), + ], + ids=lambda case: case.description, +) +def test_given_project_when_tracing_rich_lineage_with_each_engine_then_output_matches( + test_case: NativeRichLineageCliTestCase, tmp_path: Path +) -> None: + runs: dict[str, EngineLineageRun] = { + engine: engine_lineage_run( + project_dir=tmp_path / engine, + files=test_case.files, + engine=engine, + targets=test_case.lineage_targets, + mode="rich", + ) + for engine in _ENGINES + } + native: EngineLineageRun = runs["native"] + + assert native.compile_returncode == 0, native.compile_report + assert model_lineage_summaries(native.compile_report) == test_case.expected_lineage + assert [code for code, _, _ in native.traces] == [0] * len(native.traces) + assert sum('"source"' in stdout for _, stdout, _ in native.traces) >= ( + test_case.expected_minimum_traced_edges + ) + assert native.wheel_calls == test_case.expected_wheel_calls + assert runs["native-preview"] == native + + +@pytest.mark.parametrize( + "test_case", + [ + NativeRichLineageFailureCliTestCase( + description="compile --lineage-mode rich", + files=_RICH_FILES, + command=_RICH_COMPILE, + expected_returncode=1, + expected_error_line=f"_native.NativeCompilerError: {_NATIVE_PANIC_ERROR}", + expected_wheel_calls=0, + ), + NativeRichLineageFailureCliTestCase( + description="lineage --mode rich", + files=_RICH_FILES, + command=( + "--compiler-engine", + "native", + "lineage", + "order_facts.whole_amount", + "--mode", + "rich", + "--json", + ), + expected_returncode=1, + expected_error_line=f"_native.NativeCompilerError: {_NATIVE_PANIC_ERROR}", + expected_wheel_calls=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_native_rich_lineage_panic_when_running_then_command_fails_without_the_wheel( + test_case: NativeRichLineageFailureCliTestCase, tmp_path: Path +) -> None: + run: FailedLineageRun = failed_rich_lineage_run( + project_dir=tmp_path, + files=test_case.files, + args=test_case.command, + perturbation=_RICH_PANIC_INJECTION, + ) + + assert run == FailedLineageRun( + returncode=test_case.expected_returncode, + last_error_line=test_case.expected_error_line, + wheel_calls=test_case.expected_wheel_calls, + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 5f86309c58..17a30cd00a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -17,7 +17,7 @@ import zipfile from bisect import bisect_left from collections import Counter -from collections.abc import Callable, Iterator, Sequence +from collections.abc import Callable, Iterator from contextlib import contextmanager, redirect_stdout from dataclasses import replace from io import StringIO @@ -32,13 +32,12 @@ import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer import sqlbuild.cli.commands.main.project._compile as compile_command_module -import sqlbuild.cli.compile_reuse._helpers.attempt as reuse_attempt -import sqlbuild.cli.compile_reuse._helpers.project_files as reuse_project_files -import sqlbuild.cli.compile_reuse._helpers.store as reuse_store +import sqlbuild.cli.compile_reuse._helpers.native_reuse as native_reuse import sqlbuild.compiler.compile._helpers.diagnostics.recovery as diagnostic_recovery import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages import sqlbuild.compiler.compile._helpers.native_stages.sql_tests as native_sql_test_stage +import sqlbuild.compiler.contracts.main._evaluate_native_model_contracts as contract_entry import sqlbuild.compiler.contracts.main.promotion_conflicts as promotion_conflicts import sqlbuild.compiler.contracts.main.validate as contract_validation import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module @@ -50,35 +49,30 @@ ) from scripts.compiler_differential.constants import FAILURE_BASE_FILES from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter +from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.cli.compile_reuse._helpers.entry_file import ( - read_entry_header, - read_entry_stdout, - write_entry, -) from sqlbuild.cli.compile_reuse.constants import ( + EXCLUDED_ROOT_DIRECTORIES, + NATIVE_REUSE_DIRECTORY_NAME, + NATIVE_REUSE_SUFFIX, + RETIRED_REUSE_DIRECTORY_NAME, REUSE_DISABLE_ENV_VAR, - REUSE_ENTRY_DIRECTORY_NAME, -) -from sqlbuild.cli.compile_reuse.models import ( - ProjectFilesComparison, - StoredCompileHeader, - StoredCompileInputs, ) from sqlbuild.compiler.analysis_session.constants import NATIVE_ANALYSIS_STORE_FILE_NAME -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.constants import ( RETIRED_FACT_CACHE_DIRECTORY_NAME, - SQL_TEST_SCAN_STORE_FILE_NAME, + RETIRED_REFERENCE_CACHE_DIRECTORY_NAME, ) +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.frontier.constants import ( COMPILER_CACHE_DIRECTORY_NAME, COMPILER_ENGINE_ENV_VAR, RETIRED_CACHE_NAMESPACE_SUFFIXES, ) from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.observability import EventDispatcher, LifecycleEvent from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, @@ -93,9 +87,7 @@ MACRO_CALL_LOG_ENV_VAR, ) from tests.integration.src.sqlbuild.compiler.contracts.helpers import ( - NativeContractOutcome, - NativeContractRequest, - native_contract_statuses, + contract_statuses, ) _DBT_SHAPED_SQL_SIZE_PROFILE: tuple[tuple[float, int], ...] = ( @@ -2997,8 +2989,10 @@ def diagnostic_location_keys(payload: dict[str, Any]) -> tuple[tuple[str, str, i COMPILE_REUSE_REGION_ENV_VAR: "north", } COMPILE_REUSE_HIT_LINE: str = "Inputs unchanged; reused the previous compile" +REUSE_DIGESTED_FILES_KIND: str = "reuse_digested_files" _COMPILE_REUSE_RULES_CONFIG: str = ( '\n[rules]\nselect = ["XSQBR"]\n\n[rules.thresholds]\nmin_custom_rule_test_cases = 0\n' + "\n[references]\nenforce_explicit = false\n" ) _FUTURE_MTIME_OFFSET_NS: int = 5_000_000_000 _PHASE_DURATION: re.Pattern[str] = re.compile(r" \(\d+\.\d+s\)") @@ -3020,6 +3014,24 @@ def diagnostic_location_keys(payload: dict[str, Any]) -> tuple[tuple[str, str, i f"SELECT order_id, '@@ENV:{COMPILE_REUSE_REGION_ENV_VAR}' AS region " 'FROM __ref("stg_orders")\n' ), + "models/marts/_sqlbuild/_macros/order_source.py": ( + '"""Mart relation macros."""\n\n\n' + "def order_source() -> str:\n" + ' """Return the relation orders are read from."""\n' + " return '__ref(\"stg_orders\")'\n" + ), + "models/marts/referenced_orders.sql": ( + "MODEL (description 'Orders read through a direct reference.');\n\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "models/marts/macro_sourced_orders.sql": ( + "MODEL (description 'Orders read through a macro-built reference.');\n\n" + "SELECT order_id FROM @order_source()\n" + ), + "models/marts/var_sourced_orders.sql": ( + "MODEL (description 'Orders read through a variable reference.');\n\n" + 'SELECT order_id FROM __ref("@@orders_model")\n' + ), "rules/limits.py": "MAX_NAME_LENGTH: int = 40\n", "rules/naming.py": ( "from sqlbuild.rules import Finding, Model, RuleContext, rule\n\n" @@ -3056,6 +3068,12 @@ def prepare_compile_reuse_project(*, project_dir: Path) -> None: prepare_compile_cache_invalidation_project(project_dir=project_dir) for relative_path, contents in _COMPILE_REUSE_EXTRA_PROJECT_FILES.items(): write_project_file(project_dir, relative_path, contents) + replace_project_text( + project_dir, + "sqlbuild_project.toml", + 'quantity_multiplier = "2"', + 'quantity_multiplier = "2"\norders_model = "stg_orders"', + ) config_path: Path = project_dir / "sqlbuild_project.toml" config_path.write_text( config_path.read_text(encoding="utf-8") + _COMPILE_REUSE_RULES_CONFIG, encoding="utf-8" @@ -3174,55 +3192,14 @@ def recorded_events(*, path: Path) -> list[dict[str, object]]: def compile_reuse_entry_paths(*, project_dir: Path) -> tuple[Path, ...]: - """Return every stored compile reuse entry of a project.""" + """Return every stored compile reuse slot of a project.""" return tuple( - sorted((compiler_cache_directory(project_dir) / REUSE_ENTRY_DIRECTORY_NAME).glob("*.entry")) - ) - - -def _rewrite_compile_reuse_entries( - *, project_dir: Path, rewrite: Callable[[StoredCompileInputs], StoredCompileInputs] -) -> None: - for path in compile_reuse_entry_paths(project_dir=project_dir): - header: StoredCompileHeader | None = read_entry_header(path=path) - assert header is not None - stdout: str | None = read_entry_stdout(path=path, header=header) - assert stdout is not None - write_entry(path=path, inputs=rewrite(header.inputs), output=header.output, stdout=stdout) - - -def simulate_native_build_change(project_dir: Path) -> None: - """Simulate an upgraded SQLBuild version and native build in the stored identity.""" - - _rewrite_compile_reuse_entries( - project_dir=project_dir, - rewrite=lambda inputs: replace( - inputs, runtime={**inputs.runtime, "native_build": "0.0.0+simulated"} - ), - ) - - -def simulate_python_version_change(project_dir: Path) -> None: - """Simulate a different Python interpreter in the stored identity.""" - - _rewrite_compile_reuse_entries( - project_dir=project_dir, - rewrite=lambda inputs: replace( - inputs, runtime={**inputs.runtime, "python": "3.0.0 (simulated)"} - ), - ) - - -def simulate_installed_module_change(project_dir: Path) -> None: - """Simulate reinstalled packages by aging every recorded module file stamp.""" - - _rewrite_compile_reuse_entries( - project_dir=project_dir, - rewrite=lambda inputs: replace( - inputs, - modules=tuple((path, mtime_ns - 1, size) for path, mtime_ns, size in inputs.modules), - ), + sorted( + (compiler_cache_directory(project_dir) / NATIVE_REUSE_DIRECTORY_NAME).glob( + f"*{NATIVE_REUSE_SUFFIX}" + ) + ) ) @@ -3352,27 +3329,6 @@ def write_orders_api_timeout_file(project_dir: Path) -> None: ) -def stored_stdout_path(entry_path: Path) -> Path: - """Return the stored stdout file that one entry references.""" - - header: StoredCompileHeader | None = read_entry_header(path=entry_path) - assert header is not None - return entry_path.parent / header.output.stdout_file - - -def flip_stored_stdout_bytes(entry_path: Path) -> None: - """Corrupt the stored stdout without changing its length.""" - - path: Path = stored_stdout_path(entry_path) - path.write_bytes(path.read_bytes()[:-3] + b"xyz") - - -def remove_stored_stdout(entry_path: Path) -> None: - """Delete the stored stdout while keeping the entry that references it.""" - - stored_stdout_path(entry_path).unlink() - - def enable_compile_reuse(monkeypatch: pytest.MonkeyPatch) -> None: """Enable compile reuse and the fixture environment for in-process compiles.""" @@ -3419,24 +3375,24 @@ def rewrite_compiled_model_unchanged(project_dir: Path) -> None: path.write_bytes(path.read_bytes()) -def record_digested_paths(*, monkeypatch: pytest.MonkeyPatch) -> list[str]: - """Record every project file compile reuse reads to compute a content digest.""" +def record_digested_files(*, monkeypatch: pytest.MonkeyPatch) -> list[dict[str, int]]: + """Record every native reuse answer, among them the project files read for digests.""" - paths: list[str] = [] - original: Callable[..., str | None] = reuse_project_files.file_digest + reads: list[dict[str, int]] = [] + original: Callable[..., None] = native_reuse.report_native_answer - def recording_digest(*, path: str) -> str | None: - paths.append(path) - return original(path=path) + def recording(*, stage: NativeStage, kind: str, units: int = 1) -> None: + reads.append({kind: units}) + original(stage=stage, kind=kind, units=units) - monkeypatch.setattr(reuse_project_files, "file_digest", recording_digest) - return paths + monkeypatch.setattr(native_reuse, "report_native_answer", recording) + return reads def settle_racy_window(*, monkeypatch: pytest.MonkeyPatch) -> None: """Treat files written before a compile starts as settled instead of waiting two seconds.""" - monkeypatch.setattr(reuse_project_files, "RACY_WINDOW_NS", 0) + monkeypatch.setattr(native_reuse, "RACY_WINDOW_NS", 0) def write_large_file(*, project_dir: Path, relative_path: str, size_bytes: int) -> Path: @@ -3476,15 +3432,16 @@ def compile_in_process_reused( def in_process_compile_reads( - *, project_dir: Path, digested: list[str], path: Path, capsys: pytest.CaptureFixture[str] + *, project_dir: Path, digested: list[dict[str, int]], capsys: pytest.CaptureFixture[str] ) -> tuple[int, int, bool]: - """Compile in this process; return its exit code, reads of path, and whether it reused.""" + """Compile in this process; return its exit code, project files read, and whether it reused.""" start: int = len(digested) _ = capsys.readouterr() code: int = compile_in_process(project_dir=project_dir) reused: bool = COMPILE_REUSE_HIT_LINE in capsys.readouterr().err - return code, digested[start:].count(str(path)), reused + reads: int = sum(answer.get(REUSE_DIGESTED_FILES_KIND, 0) for answer in digested[start:]) + return code, reads, reused def fail_reuse_store(*, monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: @@ -3493,7 +3450,7 @@ def fail_reuse_store(*, monkeypatch: pytest.MonkeyPatch, error: BaseException) - def raise_error(**_kwargs: Any) -> None: raise error - monkeypatch.setattr(reuse_store, "provider_settings_inputs", raise_error) + monkeypatch.setattr(native_reuse, "provider_settings_inputs", raise_error) def compile_in_process_output( @@ -4173,7 +4130,7 @@ def upgrade_native_build(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: def corrupt_sql_test_scan_store(root: Path, _monkeypatch: pytest.MonkeyPatch) -> None: """Overwrite the stored SQL-test scans with bytes that are not a store file.""" - path: Path = compiler_cache_directory(root) / SQL_TEST_SCAN_STORE_FILE_NAME + path: Path = compiler_cache_directory(root) / native_module.SQL_TEST_SCAN_STORE_FILE_NAME assert path.is_file() _ = path.write_bytes(b"not a native store") @@ -4193,17 +4150,23 @@ def edit_installed_code(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: def write_retired_compiler_cache_files(project_dir: Path) -> tuple[Path, ...]: - """Leave every engine's fact cache and render files of other target slots from old releases.""" - - written: list[Path] = [ - compiler_cache_directory(project_dir).parent - / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" - / RETIRED_FACT_CACHE_DIRECTORY_NAME - / "sql-tests.sqlite3" - for suffix in RETIRED_CACHE_NAMESPACE_SUFFIXES - ] + """Leave every engine's fact and reference caches and other slots' render files behind.""" + + written: list[Path] = [] + for suffix in RETIRED_CACHE_NAMESPACE_SUFFIXES: + engine_root: Path = ( + compiler_cache_directory(project_dir).parent + / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" + ) + written.extend( + engine_root / retired_directory / "sql-tests.sqlite3" + for retired_directory in ( + RETIRED_FACT_CACHE_DIRECTORY_NAME, + RETIRED_REFERENCE_CACHE_DIRECTORY_NAME, + ) + ) written.extend( - compiler_cache_directory(project_dir) / REUSE_ENTRY_DIRECTORY_NAME / name + compiler_cache_directory(project_dir) / RETIRED_REUSE_DIRECTORY_NAME / name for name in RETIRED_RENDER_FILES ) for path in written: @@ -4212,56 +4175,62 @@ def write_retired_compiler_cache_files(project_dir: Path) -> tuple[Path, ...]: return tuple(written) -_ORIGINAL_SCAN_WRITE: Callable[..., None] = SqlTestScanCache.write +def _stored_scan_values_replaced( + replacement: bytes, +) -> Callable[[Path, pytest.MonkeyPatch], None]: + def change(root: Path, _monkeypatch: pytest.MonkeyPatch) -> None: + """Rewrite every stored scan value on disk, keeping a valid store frame and checksum.""" + path: Path = compiler_cache_directory(root) / native_module.SQL_TEST_SCAN_STORE_FILE_NAME + frame: bytes = path.read_bytes() + header: bytes = frame[:_STORE_PREFIX_LENGTH] + body: bytes = frame[_STORE_PREFIX_LENGTH + _STORE_CHECKSUM_LENGTH :] + fields: list[bytes] = [] + cursor: int = 0 + for _ in range(2): + text, cursor = _store_bytes(body, cursor) + fields.append(text) + generation: bytes = body[cursor : cursor + 8] + metadata, cursor = _store_bytes(body, cursor + 8) + count: int = int.from_bytes(body[cursor : cursor + 8], "little") + cursor += 8 + entries: list[bytes] = [] + for _ in range(count): + key, cursor = _store_bytes(body, cursor) + last_used: bytes = body[cursor : cursor + 8] + _value, cursor = _store_bytes(body, cursor + 8) + entries.append(_framed(key) + last_used + _framed(replacement)) + assert cursor == len(body) and count > 0 + rewritten: bytes = b"".join( + (*map(_framed, fields), generation, _framed(metadata), count.to_bytes(8, "little")) + ) + b"".join(entries) + checksum: bytes = hashlib.sha256(_framed(rewritten)).digest() + _ = path.write_bytes(header + checksum + rewritten) -def _stored_scans_replaced(replacement: bytes) -> Callable[[pytest.MonkeyPatch], None]: - def arrange(monkeypatch: pytest.MonkeyPatch) -> None: - def write( - self: SqlTestScanCache, - *, - algorithm: str, - syntax: SqlLexicalSyntax, - parts: Sequence[str], - value: bytes, - ) -> None: - del value - _ORIGINAL_SCAN_WRITE( - self, algorithm=algorithm, syntax=syntax, parts=parts, value=replacement - ) + return change - monkeypatch.setattr(SqlTestScanCache, "write", write) - return arrange +_STORE_PREFIX_LENGTH: int = 12 +_STORE_CHECKSUM_LENGTH: int = 32 -store_undecodable_scans: Callable[[pytest.MonkeyPatch], None] = _stored_scans_replaced(b"{]") -store_misshapen_scans: Callable[[pytest.MonkeyPatch], None] = _stored_scans_replaced( - b'[{"mode": "model"}]' -) +def _store_bytes(body: bytes, cursor: int) -> tuple[bytes, int]: + """One length-prefixed field of a native store body, and the cursor after it.""" + length: int = int.from_bytes(body[cursor : cursor + 8], "little") + return body[cursor + 8 : cursor + 8 + length], cursor + 8 + length -def ignore_test_text_in_scan_key(monkeypatch: pytest.MonkeyPatch) -> None: - """Key stored SQL-test scans by file path only, so an edited test reads a stale scan.""" - read: Callable[..., object] = SqlTestScanCache.read - write: Callable[..., None] = SqlTestScanCache.write +def _framed(value: bytes) -> bytes: + return len(value).to_bytes(8, "little") + value - def path_only(kwargs: dict[str, Any]) -> dict[str, Any]: - return {**kwargs, "parts": kwargs["parts"][:1]} - monkeypatch.setattr( - SqlTestScanCache, "read", lambda self, **kwargs: read(self, **path_only(kwargs)) - ) - monkeypatch.setattr( - SqlTestScanCache, "write", lambda self, **kwargs: write(self, **path_only(kwargs)) - ) - - -def restore_sql_test_scan_writes(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Store real scan results again after an arrangement replaced them.""" - - monkeypatch.setattr(SqlTestScanCache, "write", _ORIGINAL_SCAN_WRITE) +replace_stored_scans_undecodable: Callable[[Path, pytest.MonkeyPatch], None] = ( + _stored_scan_values_replaced(b"{]") +) +replace_stored_scans_misshapen: Callable[[Path, pytest.MonkeyPatch], None] = ( + _stored_scan_values_replaced(b'[{"mode": "model"}]') +) def keep_invalidation(_monkeypatch: pytest.MonkeyPatch) -> None: @@ -4269,12 +4238,10 @@ def keep_invalidation(_monkeypatch: pytest.MonkeyPatch) -> None: def ignore_project_changes(monkeypatch: pytest.MonkeyPatch) -> None: - """Make compile reuse believe no project file changed since the stored compile.""" + """Make compile reuse stop watching the models folder, so a model edit goes unseen.""" monkeypatch.setattr( - reuse_attempt, - "compare_project_files", - lambda **_kwargs: ProjectFilesComparison(unchanged=True, verified={}), + native_reuse, "EXCLUDED_ROOT_DIRECTORIES", EXCLUDED_ROOT_DIRECTORIES | {"models"} ) @@ -4684,7 +4651,7 @@ def freeze_macro_call_store_environment(monkeypatch: pytest.MonkeyPatch) -> None monkeypatch.setattr(macro_bridge_class, "store_environment", lambda **_kwargs: "frozen") monkeypatch.setattr(macro_bridge_class, "unchanged_module_digests", lambda **_kwargs: {}) - monkeypatch.setattr(macro_bridge_class.MacroBridge, "_observe_modules", lambda _bridge: True) + monkeypatch.setattr(macro_bridge_class.MacroBridge, "observe_modules", lambda _bridge: True) monkeypatch.setattr(call_store_module, "claim_first_compile", lambda: True) @@ -4961,10 +4928,10 @@ def type_system_engine_compile( """Compile in this process under `engine`; return the run and what native types answered.""" normalized: list[bool] = [] - contract_statuses: Counter[str] = Counter() + statuses: Counter[str] = Counter() normalize: Callable[[str, str], object] = native_module.normalize_type - evaluate_contracts: Callable[..., list[NativeContractOutcome]] = ( - native_module.evaluate_native_model_contracts + evaluate_contracts: Callable[..., ContractValidationResult] = ( + contract_entry.native_model_contracts ) def recorded(type_sql: str, dialect: str) -> object: @@ -4972,22 +4939,24 @@ def recorded(type_sql: str, dialect: str) -> object: normalized.append(result is not None) return result - def recorded_contracts(request: NativeContractRequest) -> list[NativeContractOutcome]: - outcomes: list[NativeContractOutcome] = evaluate_contracts(request) - contract_statuses.update(native_contract_statuses(request=request, outcomes=outcomes)) - return outcomes + def recorded_contracts( + *, project: CompiledProjectView, dialect: TypeDialect | str | None + ) -> ContractValidationResult: + result: ContractValidationResult = evaluate_contracts(project=project, dialect=dialect) + statuses.update(contract_statuses(project=project, result=result)) + return result type_normalization.normalize_type.cache_clear() with monkeypatch.context() as patch: patch.setattr(native_module, "normalize_type", recorded) - patch.setattr(native_module, "evaluate_native_model_contracts", recorded_contracts) + patch.setattr(contract_entry, "native_model_contracts", recorded_contracts) for name, value in {COMPILER_ENGINE_ENV_VAR: engine, REUSE_DISABLE_ENV_VAR: "1"}.items(): patch.setenv(name, value) run: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) type_normalization.normalize_type.cache_clear() return run, NativeTypeAnswers( normalized=normalized, - typed_comparisons=contract_statuses["typed_comparisons"], + typed_comparisons=statuses["typed_comparisons"], ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py index 721cb03b5a..1c3b182ce2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -46,7 +46,6 @@ fact_quantity_type_declared, fact_quantity_type_removed, ignore_project_changes, - ignore_test_text_in_scan_key, in_process_reuse_run, keep_invalidation, move_project_file, @@ -58,7 +57,8 @@ plan_between, random_edit_plan, replace_project_text, - restore_sql_test_scan_writes, + replace_stored_scans_misshapen, + replace_stored_scans_undecodable, run_reuse_compile, sql_test_scan_counts, staging_column_removed, @@ -78,8 +78,6 @@ star_chain_with_twin_added, stg_orders_test_edit, stg_orders_test_edited_again, - store_misshapen_scans, - store_undecodable_scans, twin_header_changed, upgrade_native_build, write_external_flavor, @@ -376,12 +374,6 @@ def test_given_random_edit_chain_when_compiling_with_caches_then_each_step_match sabotage=keep_invalidation, expected_matches_uncached=True, ), - BrokenEditInvalidationTestCase( - description="stale_sql_test_scan_despite_test_edit", - edit=stg_orders_test_edit, - sabotage=ignore_test_text_in_scan_key, - expected_matches_uncached=False, - ), ], ids=lambda case: case.description, ) @@ -475,14 +467,12 @@ def test_given_models_sharing_an_analysis_cache_key_when_upstream_changes_then_i ), SqlTestScanStoreTestCase( description="stored_entries_undecodable", - arrange=store_undecodable_scans, - change=restore_sql_test_scan_writes, + change=replace_stored_scans_undecodable, expected_rescans=12, ), SqlTestScanStoreTestCase( description="stored_entries_misshapen", - arrange=store_misshapen_scans, - change=restore_sql_test_scan_writes, + change=replace_stored_scans_misshapen, expected_rescans=12, ), ], @@ -537,22 +527,22 @@ def test_given_stored_sql_test_scans_when_inputs_change_then_only_changed_files_ NativeAnalysisStoreTestCase( description="staging_type_reaches_consumers", change=edit_staging_type, - expected_misses=10, + expected_misses=13, ), NativeAnalysisStoreTestCase( description="native_build_upgraded", change=upgrade_native_build, - expected_misses=14, + expected_misses=17, ), NativeAnalysisStoreTestCase( description="installed_python_code_changed", change=edit_installed_code, - expected_misses=14, + expected_misses=17, ), NativeAnalysisStoreTestCase( description="store_file_corrupted", change=corrupt_native_analysis_store, - expected_misses=14, + expected_misses=17, ), ], ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py index f107cbb22b..06dca3cc5b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py @@ -45,14 +45,12 @@ enable_compile_reuse, fail_reuse_store, flip_header_byte, - flip_stored_stdout_bytes, flip_trailing_bytes, garbage_file, in_process_compile_reads, progress_only, - record_digested_paths, + record_digested_files, recorded_events, - remove_stored_stdout, replace_project_text, rewrite_compiled_model_unchanged, rewrite_macro_without_change, @@ -60,9 +58,6 @@ run_reuse_compile_into_file, run_reuse_text_compile, settle_racy_window, - simulate_installed_module_change, - simulate_native_build_change, - simulate_python_version_change, touch_back, touch_model_without_change, truncate_file, @@ -91,6 +86,28 @@ root, "models/staging/stg_orders.sql", " quantity,", " quantity + 0 AS quantity," ), ), + CompileReuseInvalidationTestCase( + description="model_reference_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/referenced_orders.sql", + '__ref("stg_orders")', + '__ref("regional_orders")', + ), + ), + CompileReuseInvalidationTestCase( + description="macro_built_reference_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_macros/order_source.py", + '"stg_orders"', + '"regional_orders"', + ), + ), + CompileReuseInvalidationTestCase( + description="variable_reference_change", + edited_args=("--vars", json.dumps({"orders_model": "regional_orders"})), + ), CompileReuseInvalidationTestCase( description="model_added", edit=lambda root: write_project_file( @@ -219,15 +236,6 @@ CompileReuseInvalidationTestCase( description="stale_compiled_artifact_added", edit=add_stale_compiled_file ), - CompileReuseInvalidationTestCase( - description="native_build_and_version_simulated", edit=simulate_native_build_change - ), - CompileReuseInvalidationTestCase( - description="python_version_simulated", edit=simulate_python_version_change - ), - CompileReuseInvalidationTestCase( - description="installed_module_simulated", edit=simulate_installed_module_change - ), ], ids=lambda case: case.description, ) @@ -422,12 +430,6 @@ def test_given_reuse_bypass_when_compiling_twice_then_both_compiles_run_in_full( [ CompileReuseCorruptEntryTestCase(description="truncated", corrupt=truncate_file), CompileReuseCorruptEntryTestCase(description="header_tail", corrupt=flip_trailing_bytes), - CompileReuseCorruptEntryTestCase( - description="stdout_bytes", corrupt=flip_stored_stdout_bytes - ), - CompileReuseCorruptEntryTestCase( - description="stdout_missing", corrupt=remove_stored_stdout - ), CompileReuseCorruptEntryTestCase(description="header_bytes", corrupt=flip_header_byte), CompileReuseCorruptEntryTestCase(description="empty", corrupt=empty_file), CompileReuseCorruptEntryTestCase(description="garbage", corrupt=garbage_file), @@ -727,24 +729,24 @@ def test_given_large_data_file_when_storing_and_reusing_then_it_is_read_only_aft ) enable_compile_reuse(monkeypatch) settle_racy_window(monkeypatch=monkeypatch) - digested: list[str] = record_digested_paths(monkeypatch=monkeypatch) + digested: list[dict[str, int]] = record_digested_files(monkeypatch=monkeypatch) stored: tuple[int, int, bool] = in_process_compile_reads( - project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + project_dir=project_dir, digested=digested, capsys=capsys ) reused: tuple[int, int, bool] = in_process_compile_reads( - project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + project_dir=project_dir, digested=digested, capsys=capsys ) touch_back(data_file, seconds=3600) restamped: tuple[int, int, bool] = in_process_compile_reads( - project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + project_dir=project_dir, digested=digested, capsys=capsys ) touch_back(data_file, seconds=7200) verified: tuple[int, int, bool] = in_process_compile_reads( - project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + project_dir=project_dir, digested=digested, capsys=capsys ) refreshed: tuple[int, int, bool] = in_process_compile_reads( - project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + project_dir=project_dir, digested=digested, capsys=capsys ) runs: tuple[tuple[int, int, bool], ...] = (stored, reused, restamped, verified, refreshed) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 56dd2c7a3c..be3bd5dec9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -9,7 +9,6 @@ import sqlbuild._native as native_module from sqlbuild.compiler.analysis_session.models import NativeModelAnalyses -from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.planner.models import NativeSqlTestArtifact @@ -95,7 +94,9 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ assert planned_artifacts assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) planned_counts.append(len(planned_artifacts)) - assert isinstance(seams["complete_native_semantic_diagnostics"][0], CompiledProject) + assert isinstance( + seams["complete_native_semantic_diagnostics"][0], native_module.NativeProjectView + ) assert isinstance(seams["evaluate_native_model_contracts"][0], ContractValidationResult) assert [ len(call.args[0].tests) for call in native_sql_test_planning.call_args_list diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/conftest.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/conftest.py new file mode 100644 index 0000000000..a84e47530b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/conftest.py @@ -0,0 +1,19 @@ +"""Run every refactoring test under the shipped engine and the native refactoring stage.""" + +import pytest + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.types import CompilerEngine + + +@pytest.fixture( + autouse=True, + params=(CompilerEngine.NATIVE, CompilerEngine.NATIVE_PREVIEW), + ids=lambda engine: engine.value, +) +def refactor_compiler_engine( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> CompilerEngine: + engine: CompilerEngine = request.param + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + return engine diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py index 3e82dbbd6c..9f6d5ef417 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_rename_models.py @@ -30,6 +30,14 @@ write_orders_project, ) +_TAGGED_SOURCES: str = ( + "# archived feeds\n" + 'sources: [{name: archived_orders, description: "Orders # archived", ' + 'expression: !!str "(SELECT 1 AS id)"}, ' + "{name: archived_refunds, description: 'Refunds # archived', " + "expression: !!str '(SELECT 2 AS id)'}] # flow form\n" +) + @pytest.mark.parametrize( "test_case", @@ -58,6 +66,15 @@ expected_relation="stg_order_lines", extra_files=CUSTOMER_DECLARATIONS, ), + ModelRefactorE2ETestCase( + description="rename keeps tagged sources after quoted values holding ' # '", + command=("rename", "model:stg_orders", "stg_order_lines"), + expected_removed="models/staging/stg_orders.sql", + expected_file="models/staging/stg_order_lines.sql", + expected_fragments={"sources/archived.yml": (_TAGGED_SOURCES,)}, + expected_relation="stg_order_lines", + extra_files={"sources/archived.yml": _TAGGED_SOURCES}, + ), ModelRefactorE2ETestCase( description="rename rewrites reusable schema relationships audits", command=("rename", "model:stg_orders", "stg_order_lines"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/_test_types.py index fb4959db4a..08cf0c1113 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/_test_types.py @@ -43,3 +43,14 @@ class ScopeBareTargetE2eCase: target: str expected_exit_code: int expected_fragment: str + + +@dataclass(frozen=True) +class ScopeUnicodeE2eCase: + description: str + macro_names: tuple[str, ...] + glob: str + expected_available: tuple[str, ...] + expected_matched: tuple[str, ...] + expected_kind_counts: dict[str, int] + expected_exit_code: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py index 0ace9735ce..55fa9e2633 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/helpers.py @@ -77,6 +77,23 @@ def write_scope_performance_project( ) +def write_unicode_macro_project(*, project_dir: Path, macro_names: tuple[str, ...]) -> None: + """Write one model and a global macro file defining each named macro.""" + + (project_dir / "models").mkdir(parents=True) + (project_dir / "macros").mkdir() + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "scope_unicode"\nadapter = "duckdb"\n', encoding="utf-8" + ) + (project_dir / "models" / "orders.sql").write_text( + 'MODEL(description "Test model orders.");\nSELECT 1 AS id\n', encoding="utf-8" + ) + macro_source: str = "\n\n".join( + f"def {name}(expression: str) -> str:\n return expression\n" for name in macro_names + ) + (project_dir / "macros" / "names.py").write_text(macro_source, encoding="utf-8") + + def _write_scope_model(*, model_dir: Path, model_index: int, macro_name: str) -> None: (model_dir / f"model_{model_index:05d}.sql").write_text( f'MODEL(description "Test model.");\nSELECT @{macro_name}("id") AS id\n', encoding="utf-8" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py b/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py index e2e2c26196..6a73e2f76e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scope/test_scope.py @@ -11,8 +11,12 @@ from tests.e2e.src.sqlbuild.cli.commands.main.scope._test_types import ( ScopeBareTargetE2eCase, ScopeE2eCase, + ScopeUnicodeE2eCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.scope.helpers import ( + run_scope_alias, + write_unicode_macro_project, ) -from tests.e2e.src.sqlbuild.cli.commands.main.scope.helpers import run_scope_alias @pytest.mark.parametrize( @@ -131,5 +135,55 @@ def test_given_bare_name_that_is_not_a_resource_when_running_scope_then_fails_cl assert test_case.expected_fragment in result.stderr +@pytest.mark.parametrize( + "test_case", + ( + ScopeUnicodeE2eCase( + description="non-ASCII macro names", + macro_names=("zeta", "caf\u00e9", "alpha", "cafe"), + glob="caf?", + expected_available=("macro:alpha", "macro:cafe", "macro:caf\u00e9", "macro:zeta"), + expected_matched=("macro:cafe", "macro:caf\u00e9"), + expected_kind_counts={"macro": 4}, + expected_exit_code=1, + ), + ), + ids=lambda case: case.description, +) +def test_given_non_ascii_declarations_when_running_scope_then_order_and_globs_follow_code_points( + tmp_path: Path, test_case: ScopeUnicodeE2eCase +) -> None: + project_dir: Path = tmp_path / "project" + write_unicode_macro_project(project_dir=project_dir, macro_names=test_case.macro_names) + + report: subprocess.CompletedProcess[str] = run_scope_alias( + alias="sqb", project_dir=project_dir, args=("model:orders", "--json", "--globals", "all") + ) + matched: subprocess.CompletedProcess[str] = run_scope_alias( + alias="sqb", + project_dir=project_dir, + args=("model:orders", "--json", "--globals", "all", "--match", test_case.glob), + ) + browsed: subprocess.CompletedProcess[str] = run_scope_alias( + alias="sqb", + project_dir=project_dir, + args=("model:orders", "--browse", "global", "--json"), + ) + + assert report.returncode == test_case.expected_exit_code, report.stderr + assert report.stdout.isascii() + assert ( + tuple(item["identity"] for item in json.loads(report.stdout)["available"]) + == test_case.expected_available + ) + assert ( + tuple(item["identity"] for item in json.loads(matched.stdout)["available"]) + == test_case.expected_matched + ) + assert json.loads(browsed.stdout)["folders"][0]["kind_counts"] == ( + test_case.expected_kind_counts + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/goldens/compiler/failure/config-bom.json b/tests/goldens/compiler/failure/config-bom.json index e2af44d5b4..515ea740f6 100644 --- a/tests/goldens/compiler/failure/config-bom.json +++ b/tests/goldens/compiler/failure/config-bom.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml contains invalid TOML: Invalid statement (at line 1, column 1)" + "message": "/sqlbuild_project.toml contains invalid TOML: invalid statement: tomllib does not skip a byte order mark" } ] } diff --git a/tests/goldens/compiler/failure/config-toml-syntax.json b/tests/goldens/compiler/failure/config-toml-syntax.json index 3b8eaedf30..090e55ac96 100644 --- a/tests/goldens/compiler/failure/config-toml-syntax.json +++ b/tests/goldens/compiler/failure/config-toml-syntax.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml contains invalid TOML: Expected ']' at the end of a table declaration (at line 6, column 10)" + "message": "/sqlbuild_project.toml contains invalid TOML: unclosed table (at line 6)" } ] } diff --git a/tests/goldens/compiler/failure/refactor-mv-model.json b/tests/goldens/compiler/failure/refactor-mv-model.json new file mode 100644 index 0000000000..71e83421bd --- /dev/null +++ b/tests/goldens/compiler/failure/refactor-mv-model.json @@ -0,0 +1,283 @@ +{ + "commands": [ + { + "label": "refactor-mv", + "exit_code": 0, + "diagnostics": [], + "report": { + "operation": "move_model", + "target": "model:customer_totals", + "new_name": "customer_totals", + "destination": "models/finance/customer_totals.sql", + "cascade": false, + "status": "applied", + "files_changed": 1, + "files": [ + { + "path": "models/finance/customer_totals.sql", + "from_path": "models/marts/customer_totals.sql", + "edits": [] + } + ], + "migrations": [], + "renamed_columns": [], + "manual": [], + "blocking": [], + "compile": { + "ok": true, + "errors": [] + } + } + }, + { + "label": "compile", + "exit_code": 0, + "diagnostics": [] + } + ], + "compiled": { + "models/finance/customer_totals.sql": [ + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/finance/customer_totals.sql", + "original_file_path": "models/finance/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "finance", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/goldens/compiler/failure/refactor-rename-column-cascade.json b/tests/goldens/compiler/failure/refactor-rename-column-cascade.json new file mode 100644 index 0000000000..dbab17a01c --- /dev/null +++ b/tests/goldens/compiler/failure/refactor-rename-column-cascade.json @@ -0,0 +1,301 @@ +{ + "commands": [ + { + "label": "refactor-rename", + "exit_code": 0, + "diagnostics": [], + "report": { + "operation": "rename_column", + "target": "column:stg_orders.amount", + "new_name": "order_amount", + "destination": null, + "cascade": true, + "status": "applied", + "files_changed": 1, + "files": [ + { + "path": "models/staging/stg_orders.sql", + "from_path": null, + "edits": [ + { + "line": 5, + "column": 37, + "kind": "column", + "before": "", + "after": "AS order_amount" + } + ] + } + ], + "migrations": [], + "renamed_columns": [ + { + "model": "stg_orders", + "from": "amount", + "to": "order_amount" + }, + { + "model": "customer_totals", + "from": "amount", + "to": "order_amount" + } + ], + "manual": [], + "blocking": [], + "compile": { + "ok": true, + "errors": [] + } + } + }, + { + "label": "compile", + "exit_code": 0, + "diagnostics": [] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT *", + "FROM __ref(\"stg_orders\")", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount AS order_amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Every staged order", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/goldens/compiler/failure/refactor-rename-column-star-refused.json b/tests/goldens/compiler/failure/refactor-rename-column-star-refused.json new file mode 100644 index 0000000000..676dd69f70 --- /dev/null +++ b/tests/goldens/compiler/failure/refactor-rename-column-star-refused.json @@ -0,0 +1,303 @@ +{ + "commands": [ + { + "label": "refactor-rename", + "exit_code": 1, + "diagnostics": [], + "report": { + "operation": "rename_column", + "target": "column:stg_orders.amount", + "new_name": "order_amount", + "destination": null, + "cascade": false, + "status": "refused", + "files_changed": 0, + "files": [ + { + "path": "models/staging/stg_orders.sql", + "from_path": null, + "edits": [ + { + "line": 5, + "column": 37, + "kind": "column", + "before": "", + "after": "AS order_amount" + } + ] + } + ], + "migrations": [], + "renamed_columns": [ + { + "model": "stg_orders", + "from": "amount", + "to": "order_amount" + } + ], + "manual": [ + { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 1, + "reason": "SELECT * passes amount through, so this model's output column would be renamed too; rerun with --cascade or list the columns" + } + ], + "blocking": [], + "compile": { + "ok": true, + "errors": [] + } + } + }, + { + "label": "compile", + "exit_code": 0, + "diagnostics": [] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT *", + "FROM __ref(\"stg_orders\")", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Every staged order", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/goldens/compiler/failure/refactor-rename-column.json b/tests/goldens/compiler/failure/refactor-rename-column.json new file mode 100644 index 0000000000..9746f5f81c --- /dev/null +++ b/tests/goldens/compiler/failure/refactor-rename-column.json @@ -0,0 +1,310 @@ +{ + "commands": [ + { + "label": "refactor-rename", + "exit_code": 0, + "diagnostics": [], + "report": { + "operation": "rename_column", + "target": "column:stg_orders.amount", + "new_name": "order_amount", + "destination": null, + "cascade": false, + "status": "applied", + "files_changed": 2, + "files": [ + { + "path": "models/marts/customer_totals.sql", + "from_path": null, + "edits": [ + { + "line": 5, + "column": 25, + "kind": "column", + "before": "amount", + "after": "order_amount" + } + ] + }, + { + "path": "models/staging/stg_orders.sql", + "from_path": null, + "edits": [ + { + "line": 5, + "column": 37, + "kind": "column", + "before": "", + "after": "AS order_amount" + } + ] + } + ], + "migrations": [], + "renamed_columns": [ + { + "model": "stg_orders", + "from": "amount", + "to": "order_amount" + } + ], + "manual": [], + "blocking": [], + "compile": { + "ok": true, + "errors": [] + } + } + }, + { + "label": "compile", + "exit_code": 0, + "diagnostics": [] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, SUM(order_amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount AS order_amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/goldens/compiler/failure/refactor-rename-model-collision-refused.json b/tests/goldens/compiler/failure/refactor-rename-model-collision-refused.json new file mode 100644 index 0000000000..cbe35bb606 --- /dev/null +++ b/tests/goldens/compiler/failure/refactor-rename-model-collision-refused.json @@ -0,0 +1,303 @@ +{ + "commands": [ + { + "label": "refactor-rename", + "exit_code": 1, + "diagnostics": [], + "report": { + "operation": "rename_model", + "target": "model:stg_orders", + "new_name": "customer_totals", + "destination": "models/staging/customer_totals.sql", + "cascade": false, + "status": "refused", + "files_changed": 0, + "files": [ + { + "path": "models/marts/customer_totals.sql", + "from_path": null, + "edits": [ + { + "line": 6, + "column": 13, + "kind": "reference", + "before": "__ref(\"stg_orders\")", + "after": "__ref(\"customer_totals\")" + } + ] + }, + { + "path": "models/staging/customer_totals.sql", + "from_path": "models/staging/stg_orders.sql", + "edits": [] + } + ], + "migrations": [], + "renamed_columns": [], + "manual": [], + "blocking": [ + { + "path": "models/marts/customer_totals.sql", + "line": null, + "column": null, + "reason": "model:customer_totals already exists" + } + ], + "compile": { + "ok": true, + "errors": [] + } + } + }, + { + "label": "compile", + "exit_code": 0, + "diagnostics": [] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/goldens/compiler/failure/refactor-rename-model-dry-run.json b/tests/goldens/compiler/failure/refactor-rename-model-dry-run.json new file mode 100644 index 0000000000..3f50115661 --- /dev/null +++ b/tests/goldens/compiler/failure/refactor-rename-model-dry-run.json @@ -0,0 +1,296 @@ +{ + "commands": [ + { + "label": "refactor-rename", + "exit_code": 0, + "diagnostics": [], + "report": { + "operation": "rename_model", + "target": "model:stg_orders", + "new_name": "stg_order_lines", + "destination": "models/staging/stg_order_lines.sql", + "cascade": false, + "status": "dry_run", + "files_changed": 0, + "files": [ + { + "path": "models/marts/customer_totals.sql", + "from_path": null, + "edits": [ + { + "line": 6, + "column": 13, + "kind": "reference", + "before": "__ref(\"stg_orders\")", + "after": "__ref(\"stg_order_lines\")" + } + ] + }, + { + "path": "models/staging/stg_order_lines.sql", + "from_path": "models/staging/stg_orders.sql", + "edits": [] + } + ], + "migrations": [], + "renamed_columns": [], + "manual": [], + "blocking": [], + "compile": { + "ok": true, + "errors": [] + } + } + }, + { + "label": "compile", + "exit_code": 0, + "diagnostics": [] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/goldens/compiler/failure/refactor-rename-model-yaml-anchors.json b/tests/goldens/compiler/failure/refactor-rename-model-yaml-anchors.json new file mode 100644 index 0000000000..efc3507a2c --- /dev/null +++ b/tests/goldens/compiler/failure/refactor-rename-model-yaml-anchors.json @@ -0,0 +1,385 @@ +{ + "commands": [ + { + "label": "refactor-rename", + "exit_code": 0, + "diagnostics": [], + "report": { + "operation": "rename_model", + "target": "model:stg_orders", + "new_name": "stg_order_lines", + "destination": "models/staging/stg_order_lines.sql", + "cascade": false, + "status": "applied", + "files_changed": 3, + "files": [ + { + "path": "models/marts/customer_totals.sql", + "from_path": null, + "edits": [ + { + "line": 6, + "column": 13, + "kind": "reference", + "before": "__ref(\"stg_orders\")", + "after": "__ref(\"stg_order_lines\")" + } + ] + }, + { + "path": "models/staging/stg_order_lines.sql", + "from_path": "models/staging/stg_orders.sql", + "edits": [] + }, + { + "path": "sources/raw.yml", + "from_path": null, + "edits": [ + { + "line": 3, + "column": 38, + "kind": "reference", + "before": "stg_orders", + "after": "stg_order_lines" + }, + { + "line": 3, + "column": 38, + "kind": "reference", + "before": "stg_orders", + "after": "stg_order_lines" + } + ] + } + ], + "migrations": [], + "renamed_columns": [], + "manual": [], + "blocking": [], + "compile": { + "ok": true, + "errors": [] + } + } + }, + { + "label": "compile", + "exit_code": 0, + "diagnostics": [] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_order_lines\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_order_lines.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_order_lines" + ] + } + }, + "model.failure_corpus.stg_order_lines": { + "database": null, + "schema": "main", + "name": "stg_order_lines", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_order_lines.sql", + "original_file_path": "models/staging/stg_order_lines.sql", + "unique_id": "model.failure_corpus.stg_order_lines", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_order_lines" + ], + "alias": "stg_order_lines", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_order_lines", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Feeds __ref('stg_order_lines').", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + }, + "source.failure_corpus.raw_orders_copy": { + "database": null, + "schema": "", + "name": "raw_orders_copy", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders_copy", + "fqn": [ + "failure_corpus", + "raw_orders_copy" + ], + "source_name": "raw_orders_copy", + "source_description": "", + "loader": "", + "identifier": "raw_orders_copy", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Feeds __ref('stg_order_lines').", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_order_lines" + ], + "model.failure_corpus.stg_order_lines": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [], + "source.failure_corpus.raw_orders_copy": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/goldens/compiler/failure/refactor-rename-model.json b/tests/goldens/compiler/failure/refactor-rename-model.json new file mode 100644 index 0000000000..ecb2e359f0 --- /dev/null +++ b/tests/goldens/compiler/failure/refactor-rename-model.json @@ -0,0 +1,317 @@ +{ + "commands": [ + { + "label": "refactor-rename", + "exit_code": 0, + "diagnostics": [], + "report": { + "operation": "rename_model", + "target": "model:stg_orders", + "new_name": "stg_order_lines", + "destination": "models/staging/stg_order_lines.sql", + "cascade": false, + "status": "applied", + "files_changed": 2, + "files": [ + { + "path": "models/marts/customer_totals.sql", + "from_path": null, + "edits": [ + { + "line": 2, + "column": 55, + "kind": "reference", + "before": "stg_orders", + "after": "stg_order_lines" + }, + { + "line": 7, + "column": 13, + "kind": "reference", + "before": "__ref(\"stg_orders\")", + "after": "__ref(\"stg_order_lines\")" + } + ] + }, + { + "path": "models/staging/stg_order_lines.sql", + "from_path": "models/staging/stg_orders.sql", + "edits": [ + { + "line": 1, + "column": 8, + "kind": "migration", + "before": "", + "after": "migrate_from stg_orders" + } + ] + } + ], + "migrations": [ + { + "model": "stg_order_lines", + "declaration": "migrate_from stg_orders", + "reason": "keeps the relation's history and its old name working" + } + ], + "renamed_columns": [], + "manual": [], + "blocking": [], + "compile": { + "ok": true, + "errors": [] + } + } + }, + { + "label": "compile", + "exit_code": 0, + "diagnostics": [] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_order_lines\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_order_lines.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "table", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer, from __ref('stg_order_lines')", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_order_lines" + ] + } + }, + "model.failure_corpus.stg_order_lines": { + "database": null, + "schema": "main", + "name": "stg_order_lines", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_order_lines.sql", + "original_file_path": "models/staging/stg_order_lines.sql", + "unique_id": "model.failure_corpus.stg_order_lines", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_order_lines" + ], + "alias": "stg_order_lines", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "table", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_order_lines", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_order_lines" + ], + "model.failure_corpus.stg_order_lines": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json new file mode 100644 index 0000000000..79fec18808 --- /dev/null +++ b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json @@ -0,0 +1,30 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "rule", + "severity": "error", + "code": "SQBRSQL005", + "message": "CTE is unreachable from the final query", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 114, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 114 + }, + "help": "Reference the CTE from the final query or another reachable CTE, or remove it.", + "notes": [ + "Fix available: `sqb format --fix` applies it after compiler verification" + ] + } + ] + } + ], + "compiled": {}, + "manifest": null +} diff --git a/tests/goldens/compiler/seed/0-postgres.json b/tests/goldens/compiler/seed/0-postgres.json index 13f05d651d..9acbefd031 100644 --- a/tests/goldens/compiler/seed/0-postgres.json +++ b/tests/goldens/compiler/seed/0-postgres.json @@ -4,6 +4,11 @@ "label": "compile", "exit_code": 0, "diagnostics": [] + }, + { + "label": "compile-rich-lineage", + "exit_code": 0, + "diagnostics": [] } ], "compiled": { diff --git a/tests/goldens/compiler/seed/0-snowflake.json b/tests/goldens/compiler/seed/0-snowflake.json index 86d39efb09..0c3ae2f51c 100644 --- a/tests/goldens/compiler/seed/0-snowflake.json +++ b/tests/goldens/compiler/seed/0-snowflake.json @@ -4,6 +4,11 @@ "label": "compile", "exit_code": 0, "diagnostics": [] + }, + { + "label": "compile-rich-lineage", + "exit_code": 0, + "diagnostics": [] } ], "compiled": { diff --git a/tests/goldens/compiler/seed/0.json b/tests/goldens/compiler/seed/0.json index 3446f321b7..e33a3c626f 100644 --- a/tests/goldens/compiler/seed/0.json +++ b/tests/goldens/compiler/seed/0.json @@ -29,6 +29,271 @@ "label": "compile-rich-lineage", "exit_code": 0, "diagnostics": [] + }, + { + "label": "lineage-rich-upstream", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "direction": "upstream", + "metadata": { + "mode": "rich", + "max_depth": null, + "analyzed_models": 2, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "source", + "resource_name": "raw_events_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + } + ] + } + }, + { + "label": "lineage-rich-downstream", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "target": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "direction": "downstream", + "metadata": { + "mode": "rich", + "max_depth": null, + "analyzed_models": 5, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "append_stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "fulfillment_summary_2", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "fulfillment_summary_2", + "column_name": "adjust_2_amount" + }, + "transform": "expression", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "bounded_stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + } + ] + } + }, + { + "label": "lineage-fast-upstream", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "direction": "upstream", + "metadata": { + "mode": "fast", + "max_depth": null, + "analyzed_models": 2, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "source", + "resource_name": "raw_events_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + } + ] + } + }, + { + "label": "lineage-relations", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "nodes": [ + { + "id": "model:append_stg_fulfillment_1", + "name": "append_stg_fulfillment_1", + "resource_type": "model", + "relative_path": "models/features/append_stg_fulfillment_1.sql", + "qualified_name": "ci_checks.append_stg_fulfillment_1" + }, + { + "id": "model:bounded_stg_fulfillment_1", + "name": "bounded_stg_fulfillment_1", + "resource_type": "model", + "relative_path": "models/rendering/bounded_stg_fulfillment_1.sql", + "qualified_name": "ci_checks.bounded_stg_fulfillment_1" + }, + { + "id": "model:cte_facts_stg_fulfillment_1", + "name": "cte_facts_stg_fulfillment_1", + "resource_type": "model", + "relative_path": "models/analysis/cte_facts_stg_fulfillment_1.sql", + "qualified_name": "ci_checks.cte_facts_stg_fulfillment_1" + }, + { + "id": "model:fulfillment_summary_2", + "name": "fulfillment_summary_2", + "resource_type": "model", + "relative_path": "models/fulfillment/marts/fulfillment_summary_2.sql", + "qualified_name": "ci_checks.fulfillment_summary_2" + }, + { + "id": "model:stg_fulfillment_1", + "name": "stg_fulfillment_1", + "resource_type": "model", + "relative_path": "models/fulfillment/staging/stg_fulfillment_1.sql", + "qualified_name": "ci_checks.stg_fulfillment_1" + }, + { + "id": "source:raw_events_0", + "name": "raw_events_0", + "resource_type": "source", + "relative_path": "sources/raw.yml" + } + ], + "edges": [ + { + "from": "model:stg_fulfillment_1", + "to": "model:append_stg_fulfillment_1" + }, + { + "from": "model:stg_fulfillment_1", + "to": "model:bounded_stg_fulfillment_1" + }, + { + "from": "model:stg_fulfillment_1", + "to": "model:cte_facts_stg_fulfillment_1" + }, + { + "from": "model:stg_fulfillment_1", + "to": "model:fulfillment_summary_2" + }, + { + "from": "source:raw_events_0", + "to": "model:stg_fulfillment_1" + } + ], + "focus": [ + "model:stg_fulfillment_1" + ], + "direction": "both" + } } ], "compiled": { diff --git a/tests/goldens/compiler/seed/10.json b/tests/goldens/compiler/seed/10.json index 4ca491f751..87c167c0af 100644 --- a/tests/goldens/compiler/seed/10.json +++ b/tests/goldens/compiler/seed/10.json @@ -29,6 +29,307 @@ "label": "compile-rich-lineage", "exit_code": 0, "diagnostics": [] + }, + { + "label": "lineage-rich-upstream", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_sales_0", + "column_name": "amount" + }, + "direction": "upstream", + "metadata": { + "mode": "rich", + "max_depth": null, + "analyzed_models": 2, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_sales_0", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "source", + "resource_name": "raw_events_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + } + ] + } + }, + { + "label": "lineage-rich-downstream", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "target": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "direction": "downstream", + "metadata": { + "mode": "rich", + "max_depth": null, + "analyzed_models": 7, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_sales_0", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "merge_stg_sales_0", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "spaced_stg_sales_0", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "taxed_stg_sales_0", + "column_name": "taxed" + }, + "transform": "expression", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "taxed_stg_sales_0", + "column_name": "rounded" + }, + "transform": "expression", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "sales_summary_0", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + } + ] + } + }, + { + "label": "lineage-fast-upstream", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_sales_0", + "column_name": "amount" + }, + "direction": "upstream", + "metadata": { + "mode": "fast", + "max_depth": null, + "analyzed_models": 2, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_sales_0", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "medium" + }, + { + "source": { + "resource_type": "source", + "resource_name": "raw_events_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "stg_sales_0", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + } + ] + } + }, + { + "label": "lineage-relations", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "nodes": [ + { + "id": "model:copied_stg_sales_0", + "name": "copied_stg_sales_0", + "resource_type": "model", + "relative_path": "models/features/copied_stg_sales_0.sql", + "qualified_name": "main.copied_stg_sales_0" + }, + { + "id": "model:cte_facts_stg_sales_0", + "name": "cte_facts_stg_sales_0", + "resource_type": "model", + "relative_path": "models/analysis/cte_facts_stg_sales_0.sql", + "qualified_name": "main.cte_facts_stg_sales_0" + }, + { + "id": "model:merge_stg_sales_0", + "name": "merge_stg_sales_0", + "resource_type": "model", + "relative_path": "models/features/merge_stg_sales_0.sql", + "qualified_name": "main.merge_stg_sales_0" + }, + { + "id": "model:sales_summary_0", + "name": "sales_summary_0", + "resource_type": "model", + "relative_path": "models/sales/marts/sales_summary_0.sql", + "qualified_name": "main.sales_summary_0" + }, + { + "id": "model:spaced_stg_sales_0", + "name": "spaced_stg_sales_0", + "resource_type": "model", + "relative_path": "models/features/spaced_stg_sales_0.sql", + "qualified_name": "main.spaced_stg_sales_0" + }, + { + "id": "model:stg_sales_0", + "name": "stg_sales_0", + "resource_type": "model", + "relative_path": "models/sales/staging/stg_sales_0.sql", + "qualified_name": "main.stg_sales_0" + }, + { + "id": "model:taxed_stg_sales_0", + "name": "taxed_stg_sales_0", + "resource_type": "model", + "relative_path": "models/features/taxed_stg_sales_0.sql", + "qualified_name": "main.taxed_stg_sales_0" + }, + { + "id": "source:raw_events_1", + "name": "raw_events_1", + "resource_type": "source", + "relative_path": "sources/raw.yml" + } + ], + "edges": [ + { + "from": "model:stg_sales_0", + "to": "model:copied_stg_sales_0" + }, + { + "from": "model:stg_sales_0", + "to": "model:cte_facts_stg_sales_0" + }, + { + "from": "model:stg_sales_0", + "to": "model:merge_stg_sales_0" + }, + { + "from": "model:stg_sales_0", + "to": "model:sales_summary_0" + }, + { + "from": "model:stg_sales_0", + "to": "model:spaced_stg_sales_0" + }, + { + "from": "source:raw_events_1", + "to": "model:stg_sales_0" + }, + { + "from": "model:stg_sales_0", + "to": "model:taxed_stg_sales_0" + } + ], + "focus": [ + "model:stg_sales_0" + ], + "direction": "both" + } } ], "compiled": { diff --git a/tests/goldens/compiler/seed/2.json b/tests/goldens/compiler/seed/2.json index 67c315c293..1bd8afb7da 100644 --- a/tests/goldens/compiler/seed/2.json +++ b/tests/goldens/compiler/seed/2.json @@ -77,6 +77,86 @@ "message": "Model file models/sales/marts/sales_summary_1.sql references unknown model 'missing_model'" } ] + }, + { + "label": "lineage-rich-upstream", + "exit_code": 1, + "diagnostics": [ + "Project discovery START", + "Project assembly START", + "Declaration parsing START", + "Declaration parsing OK ()", + "Python discovery START", + "Python discovery OK ()", + "Project assembly OK ()", + "Project discovery OK ()", + "error[P001]: Model file models/sales/marts/sales_summary_1.sql references unknown model 'missing_model'", + "", + "" + ], + "stdout": [ + "" + ] + }, + { + "label": "lineage-rich-downstream", + "exit_code": 1, + "diagnostics": [ + "Project discovery START", + "Project assembly START", + "Declaration parsing START", + "Declaration parsing OK ()", + "Python discovery START", + "Python discovery OK ()", + "Project assembly OK ()", + "Project discovery OK ()", + "error[P001]: Model file models/sales/marts/sales_summary_1.sql references unknown model 'missing_model'", + "", + "" + ], + "stdout": [ + "" + ] + }, + { + "label": "lineage-fast-upstream", + "exit_code": 1, + "diagnostics": [ + "Project discovery START", + "Project assembly START", + "Declaration parsing START", + "Declaration parsing OK ()", + "Python discovery START", + "Python discovery OK ()", + "Project assembly OK ()", + "Project discovery OK ()", + "error[P001]: Model file models/sales/marts/sales_summary_1.sql references unknown model 'missing_model'", + "", + "" + ], + "stdout": [ + "" + ] + }, + { + "label": "lineage-relations", + "exit_code": 1, + "diagnostics": [ + "Project discovery START", + "Project assembly START", + "Declaration parsing START", + "Declaration parsing OK ()", + "Python discovery START", + "Python discovery OK ()", + "Project assembly OK ()", + "Project discovery OK ()", + "error[P001]: Model file models/sales/marts/sales_summary_1.sql references unknown model 'missing_model'", + "", + "" + ], + "stdout": [ + "" + ] } ], "compiled": {}, diff --git a/tests/goldens/compiler/seed/6.json b/tests/goldens/compiler/seed/6.json index 98fe263c63..2608dcf01b 100644 --- a/tests/goldens/compiler/seed/6.json +++ b/tests/goldens/compiler/seed/6.json @@ -29,6 +29,207 @@ "label": "compile-rich-lineage", "exit_code": 0, "diagnostics": [] + }, + { + "label": "lineage-rich-upstream", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "direction": "upstream", + "metadata": { + "mode": "rich", + "max_depth": null, + "analyzed_models": 2, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "source", + "resource_name": "raw_events_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + } + ] + } + }, + { + "label": "lineage-rich-downstream", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "target": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "direction": "downstream", + "metadata": { + "mode": "rich", + "max_depth": null, + "analyzed_models": 3, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "audited_stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + } + ] + } + }, + { + "label": "lineage-fast-upstream", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "direction": "upstream", + "metadata": { + "mode": "fast", + "max_depth": null, + "analyzed_models": 2, + "truncated": false + }, + "trace": [ + { + "source": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "cte_facts_stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + }, + { + "source": { + "resource_type": "source", + "resource_name": "raw_events_0", + "column_name": "amount" + }, + "target": { + "resource_type": "model", + "resource_name": "stg_fulfillment_1", + "column_name": "amount" + }, + "transform": "direct", + "confidence": "high" + } + ] + } + }, + { + "label": "lineage-relations", + "exit_code": 0, + "diagnostics": [], + "stdout": { + "nodes": [ + { + "id": "model:audited_stg_fulfillment_1", + "name": "audited_stg_fulfillment_1", + "resource_type": "model", + "relative_path": "models/features/audited_stg_fulfillment_1.sql", + "qualified_name": "main.audited_stg_fulfillment_1" + }, + { + "id": "model:cte_facts_stg_fulfillment_1", + "name": "cte_facts_stg_fulfillment_1", + "resource_type": "model", + "relative_path": "models/analysis/cte_facts_stg_fulfillment_1.sql", + "qualified_name": "main.cte_facts_stg_fulfillment_1" + }, + { + "id": "model:stg_fulfillment_1", + "name": "stg_fulfillment_1", + "resource_type": "model", + "relative_path": "models/fulfillment/staging/stg_fulfillment_1.sql", + "qualified_name": "main.stg_fulfillment_1" + }, + { + "id": "source:raw_events_0", + "name": "raw_events_0", + "resource_type": "source", + "relative_path": "sources/raw.yml" + } + ], + "edges": [ + { + "from": "model:stg_fulfillment_1", + "to": "model:audited_stg_fulfillment_1" + }, + { + "from": "model:stg_fulfillment_1", + "to": "model:cte_facts_stg_fulfillment_1" + }, + { + "from": "source:raw_events_0", + "to": "model:stg_fulfillment_1" + } + ], + "focus": [ + "model:stg_fulfillment_1" + ], + "direction": "both" + } } ], "compiled": { diff --git a/tests/goldens/python_oracle/declaration_usages_usage_scan_edges.json b/tests/goldens/python_oracle/declaration_usages_usage_scan_edges.json deleted file mode 100644 index 431e8234f4..0000000000 --- a/tests/goldens/python_oracle/declaration_usages_usage_scan_edges.json +++ /dev/null @@ -1,86 +0,0 @@ -[ -[ -"declaration_usages", -"6adc2535c49443cb6b3cc3cd" -], -[ -"declaration_usages", -"6070cec0a6fcc43f8c7ab528" -], -[ -"declaration_usages", -"6786f18f6ff42aa577c99d86" -], -[ -"declaration_usages", -"6070cec0a6fcc43f8c7ab528" -], -[ -"declaration_usages", -"7851fd987ffcb6afe0244d4d" -], -[ -"declaration_usages", -"6070cec0a6fcc43f8c7ab528" -], -[ -"declaration_usages", -"6070cec0a6fcc43f8c7ab528" -], -[ -"declaration_usages", -"7851fd987ffcb6afe0244d4d" -], -[ -"declaration_usages", -"6070cec0a6fcc43f8c7ab528" -], -[ -"declaration_usages", -"99c78fa41fe002d8642754e5" -], -[ -"declaration_usages", -"6786f18f6ff42aa577c99d86" -], -[ -"declaration_usages", -"ac6c4233f9df675ec4b5f9fd" -], -[ -"declaration_usages", -"6786f18f6ff42aa577c99d86" -], -[ -"declaration_usages", -"6786f18f6ff42aa577c99d86" -], -[ -"declaration_usages", -"6adc2535c49443cb6b3cc3cd" -], -[ -"declaration_usages", -"6786f18f6ff42aa577c99d86" -], -[ -"declaration_usages", -"077a3f290320a2b8cea483fc" -], -[ -"declaration_usages", -"6f95d4337ea1069397695fc8" -], -[ -"declaration_usages", -"6f95d4337ea1069397695fc8" -], -[ -"declaration_usages", -"6f95d4337ea1069397695fc8" -], -[ -"declaration_usages", -"077a3f290320a2b8cea483fc" -] -] diff --git a/tests/goldens/python_oracle/helper_reads_helper_cte_read_edges.json b/tests/goldens/python_oracle/helper_reads_helper_cte_read_edges.json deleted file mode 100644 index 0bdad0d555..0000000000 --- a/tests/goldens/python_oracle/helper_reads_helper_cte_read_edges.json +++ /dev/null @@ -1,58 +0,0 @@ -[ -[ -"helper_reads", -"aa8cdea031c78169cfcfc8cb" -], -[ -"helper_reads", -"7af39b7194de7feac7d049f8" -], -[ -"helper_reads", -"6786f18f6ff42aa577c99d86" -], -[ -"helper_reads", -"77d5413fc4891929962834c2" -], -[ -"helper_reads", -"aa8cdea031c78169cfcfc8cb" -], -[ -"helper_reads", -"6786f18f6ff42aa577c99d86" -], -[ -"helper_reads", -"6786f18f6ff42aa577c99d86" -], -[ -"helper_reads", -"74234e98afe7498fb5daf1f3" -], -[ -"helper_reads", -"4e5d6ec7534ff80d3cf04e61" -], -[ -"helper_reads", -"aa8cdea031c78169cfcfc8cb" -], -[ -"helper_reads", -"ed29b0da213826917a1d4a41" -], -[ -"helper_reads", -"6786f18f6ff42aa577c99d86" -], -[ -"helper_reads", -"74234e98afe7498fb5daf1f3" -], -[ -"helper_reads", -"77d5413fc4891929962834c2" -] -] diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py index 8b9c4aad86..96ae2437c0 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py @@ -25,3 +25,12 @@ class BareTargetRefactorTestCase: expected_path: str expected_fragment: str expected_missing_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class NativeEditCountTestCase: + description: str + command: str + target: str + new_name: str | None + expected_native_counts: dict[str, dict[str, int]] diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/conftest.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/conftest.py new file mode 100644 index 0000000000..a84e47530b --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/conftest.py @@ -0,0 +1,19 @@ +"""Run every refactoring test under the shipped engine and the native refactoring stage.""" + +import pytest + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.types import CompilerEngine + + +@pytest.fixture( + autouse=True, + params=(CompilerEngine.NATIVE, CompilerEngine.NATIVE_PREVIEW), + ids=lambda engine: engine.value, +) +def refactor_compiler_engine( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> CompilerEngine: + engine: CompilerEngine = request.param + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + return engine diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py new file mode 100644 index 0000000000..e9acb096e7 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py @@ -0,0 +1,43 @@ +"""Native work counters for refactor command integration tests.""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +import pytest + +from sqlbuild.compiler.frontier._helpers.process_recorder import process_recorder +from sqlbuild.compiler.frontier.classes.native_fallback_recorder import NativeFallbackRecorder +from sqlbuild.compiler.frontier.constants import NATIVE_ANSWER_SITE_SUFFIX +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR + +_REFACTORING_ANSWERS: str = f"{NativeStage.REFACTORING.value}{NATIVE_ANSWER_SITE_SUFFIX}" +_ANSWER_KINDS: tuple[str, ...] = ("planned_edits", "migration_edits") + + +def start_recording(*, record_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Record native answers of this process below `record_dir` until `refactoring_answers`.""" + + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + process_recorder.cache_clear() + + +def refactoring_answers(*, record_dir: Path) -> dict[str, int]: + """Write the recorded counts and return the refactoring stage's native answer counts by kind.""" + + recorder: NativeFallbackRecorder | None = process_recorder() + assert recorder is not None + recorder.write() + process_recorder.cache_clear() + counts: Counter[tuple[str, str]] = Counter() + for path in sorted(record_dir.glob("native-fallbacks-*.json")): + counts.update( + { + (str(site), str(kind)): int(count) + for site, kind, count in json.loads(path.read_text(encoding="utf-8"))["fallbacks"] + } + ) + return {kind: counts[(_REFACTORING_ANSWERS, kind)] for kind in _ANSWER_KINDS} diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py index db5666b4cf..fca7608903 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/refactor/test_command.py @@ -9,10 +9,16 @@ from sqlbuild.cli.commands._helpers.refactor.command import run_refactor_command from sqlbuild.cli.commands.models import RefactorCommandRequest from sqlbuild.cli.commands.types import CliCommand +from sqlbuild.compiler.frontier.types import CompilerEngine from tests.integration.src.sqlbuild.cli.commands._helpers.refactor._test_types import ( BareTargetRefactorTestCase, + NativeEditCountTestCase, RefactorStatusMessagesTestCase, ) +from tests.integration.src.sqlbuild.cli.commands._helpers.refactor.helpers import ( + refactoring_answers, + start_recording, +) _PROJECT_FILES: dict[str, str] = { "sqlbuild_project.toml": ( @@ -171,5 +177,64 @@ def test_given_bare_target_when_running_then_applies_refactor( assert not (tmp_path / missing_path).exists() +@pytest.mark.parametrize( + "test_case", + [ + NativeEditCountTestCase( + description="a model rename plans its reference natively and adds migrate_from", + command=CliCommand.RENAME, + target="model:stg_orders", + new_name="stg_order_lines", + expected_native_counts={ + CompilerEngine.NATIVE.value: {"planned_edits": 1, "migration_edits": 1}, + CompilerEngine.NATIVE_PREVIEW.value: {"planned_edits": 1, "migration_edits": 1}, + }, + ), + NativeEditCountTestCase( + description="a column rename plans the owner output and the downstream alias natively", + command=CliCommand.RENAME, + target="column:stg_orders.amount", + new_name="revenue", + expected_native_counts={ + CompilerEngine.NATIVE.value: {"planned_edits": 3, "migration_edits": 0}, + CompilerEngine.NATIVE_PREVIEW.value: {"planned_edits": 3, "migration_edits": 0}, + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_refactor_when_running_then_native_stage_plans_exact_edit_count( + test_case: NativeEditCountTestCase, + refactor_compiler_engine: CompilerEngine, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = tmp_path / "project" + relative_path: str + content: str + for relative_path, content in _PROJECT_FILES.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + start_recording(record_dir=tmp_path / "records", monkeypatch=monkeypatch) + + exit_code: int = run_refactor_command( + request=RefactorCommandRequest( + command=test_case.command, + target=test_case.target, + new_name=test_case.new_name, + project_dir=project_dir, + no_color=True, + ) + ) + + assert exit_code == 0, capsys.readouterr().out + assert ( + refactoring_answers(record_dir=tmp_path / "records") + == test_case.expected_native_counts[refactor_compiler_engine.value] + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index a4be012149..a9e668eada 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -147,6 +147,16 @@ class CombinedCompilationTestCase: expected_diagnostics: tuple[str, ...] +@dataclass(frozen=True) +class EarlyLintInputsTestCase: + """Early SQL lint: the expansions it took from the compile and redid.""" + + description: str + expected_findings: tuple[tuple[str, str, int | None, int | None], ...] + expected_compile_expansions: int + expected_python_expansions: int + + @dataclass(frozen=True) class RulesIntegrationTestCase: """One compiler-integrated Rules command expectation.""" @@ -1043,6 +1053,19 @@ class DefaultConnectionCliTestCase: expected_order_rows: tuple[tuple[object, ...], ...] | None = None +@dataclass(frozen=True) +class LocalOverrideCliTestCase: + """One build whose target, vars and connection the local file and CLI override.""" + + description: str + local_toml: str + argv: tuple[str, ...] + expected_exit_code: int + expected_database: str + expected_schema: str + expected_order_rows: tuple[tuple[object, ...], ...] + + @dataclass(frozen=True) class InferredFreshnessCliTestCase: """One source whose freshness strategy and type come from its declaration.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/conftest.py b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/conftest.py new file mode 100644 index 0000000000..a84e47530b --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/column_migrations/conftest.py @@ -0,0 +1,19 @@ +"""Run every refactoring test under the shipped engine and the native refactoring stage.""" + +import pytest + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.types import CompilerEngine + + +@pytest.fixture( + autouse=True, + params=(CompilerEngine.NATIVE, CompilerEngine.NATIVE_PREVIEW), + ids=lambda engine: engine.value, +) +def refactor_compiler_engine( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> CompilerEngine: + engine: CompilerEngine = request.param + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + return engine diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/_test_types.py new file mode 100644 index 0000000000..35410706f9 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/_test_types.py @@ -0,0 +1,79 @@ +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import pytest + + +@dataclass(frozen=True) +class CompileOutputsSequenceTestCase: + """A cached edit sequence and the native output work counted per step.""" + + description: str + expected_exit_codes: tuple[int, ...] + expected_work: tuple[dict[str, int], ...] + + +@dataclass(frozen=True) +class PublicationFailureTestCase: + """A staged publication onto a path a directory blocks.""" + + description: str + expected_error: str + expected_published_files: int + + +@dataclass(frozen=True) +class ReuseDisruptionTestCase: + """A stored compile disrupted between an unchanged rerun and the next compile.""" + + description: str + disrupt: Callable[[Path, pytest.MonkeyPatch], None] + expected_reused: tuple[bool, ...] + expected_slots: int + + +@dataclass(frozen=True) +class JsonReportTextTestCase: + """One report value and the text the shipped encoders wrote for it.""" + + description: str + report: dict[object, object] + expected_text: str + + +@dataclass(frozen=True) +class JsonReportErrorTestCase: + """One report value the shipped encoder rejected, and its error message.""" + + description: str + report: dict[object, object] + orjson_only: bool + expected_message: str + + +@dataclass(frozen=True) +class CliJsonReportTestCase: + """One compile of the surrogate project and the report text or error it produces.""" + + description: str + args: tuple[str, ...] + expected_fragment: str + + +@dataclass(frozen=True) +class NestedReportTestCase: + """A report nested `depth` lists deep under one key, beyond what `json.dumps` can write.""" + + description: str + depth: int + expected_error: type[Exception] + + +@dataclass(frozen=True) +class NestedHookReportTestCase: + """A real compile whose Python hook payload nests `depth` lists inside the report.""" + + description: str + depth: int + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py new file mode 100644 index 0000000000..c940178260 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -0,0 +1,403 @@ +import importlib +import json +import os +import re +import shutil +import sys +from collections import Counter +from collections.abc import Callable +from pathlib import Path +from typing import Any, NamedTuple, cast + +import pytest + +import sqlbuild._native as native_module +from sqlbuild.cli.commands._helpers.compile import output as output_module +from sqlbuild.cli.commands._helpers.compile import target_writer as target_writer_module +from sqlbuild.cli.commands.classes import native_artifact_batch as artifact_batch_module +from sqlbuild.cli.commands.classes import prepared_compile_artifacts +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.cli.compile_reuse._helpers import native_reuse as native_reuse_module +from sqlbuild.cli.compile_reuse._helpers.runtime_identity import runtime_identity +from sqlbuild.cli.compile_reuse.constants import ( + NATIVE_REUSE_DIRECTORY_NAME, + NATIVE_REUSE_SUFFIX, + REUSE_DISABLE_ENV_VAR, +) +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild.compiler.frontier.types import NativeStage + +STAGING_PREFIX: str = ".sqlbuild-staging-" +OUTPUT_MODEL_HEADER: str = ( + "MODEL (description 'Orders {name}.', materialized table, contract enforced, " + "columns (order_id (type INTEGER)));\n" +) +OUTPUT_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL021"]\n' + ), + **{ + f"models/orders_{name}.sql": OUTPUT_MODEL_HEADER.format(name=name) + + "SELECT CAST(1 AS INTEGER) AS order_id\n" + for name in ("east", "north", "west") + }, +} +EDITED_MODEL: str = "models/orders_north.sql" +DELETED_MODEL: str = "models/orders_west.sql" +COMPILE_OUTPUTS_PREFIX: str = f"{NativeStage.COMPILE_OUTPUTS.value}:" +TOUCH_BACK_NS: int = 3_600_000_000_000 +BLOCKED_ARTIFACT: str = "target/compiled/models/orders_north.sql" +HELPER_MODULE_DIRECTORY: str = "site_packages" +HELPER_MODULE: str = "orders_reuse_helper" +TRACKED_VARIABLE: str = "SQLBUILD_ORDERS_REGION" + +SURROGATE_SELECTOR: str = "orders\udcff" +SURROGATE_HOOK_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n", + "hooks/python/notify.py": ( + "from sqlbuild.hooks import hook\n\n\n@hook\ndef notify_complete(ctx):\n" + ' """Notify "q" \\\\ café \\udcff\\U0001F600\\ud83d\\ude00\\ud83d end."""\n' + " return None\n" + ), +} + + +def nested_payload_hook_files(*, depth: int) -> dict[str, str]: + """A model whose Python post-hook payload is `depth` nested lists around 1.""" + + return { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table, post_hooks [python(\"notify\", " + f"payload: {'[' * depth}1{']' * depth})]);\nSELECT 1 AS order_id\n" + ), + "hooks/python/notify.py": ( + "from sqlbuild.hooks import hook\n\n\n@hook\ndef notify(ctx, payload):\n" + ' """Notify."""\n return None\n' + ), + } + + +def nested_lists(*, depth: int) -> object: + """`depth` nested single-item lists around 1.""" + + value: object = 1 + for _ in range(depth): + value = [value] + return value + + +type CompileOutcome = tuple[int, dict[str, Any], dict[str, bytes]] + + +def write_files(*, project_dir: Path, files: dict[str, str]) -> None: + """Write project files at their project-relative paths.""" + + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + +def record_output_work(*, monkeypatch: pytest.MonkeyPatch, reuse_disabled: str) -> Counter[str]: + """Count native compile output work, staging small projects and settling file stamps.""" + + counts: Counter[str] = Counter() + monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, reuse_disabled) + monkeypatch.setattr(prepared_compile_artifacts, "_MIN_PREPARED_ARTIFACT_MODELS", 1) + monkeypatch.setattr(native_reuse_module, "RACY_WINDOW_NS", 0) + + def counted(*, stage: NativeStage, kind: str, units: int = 1) -> None: + counts[f"{stage.value}:{kind}"] += units + + for module in ( + output_module, + target_writer_module, + artifact_batch_module, + native_reuse_module, + ): + monkeypatch.setattr(module, "report_native_answer", counted) + return counts + + +def main_compile_json(*, project_dir: Path) -> int: + """Compile through the CLI with `--json --no-cache`; the report stays in captured stdout.""" + + return main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) + + +def compile_json_text( + *, project_dir: Path, args: tuple[str, ...], capsys: pytest.CaptureFixture[str] +) -> str: + """Compile through the CLI and return the JSON report text.""" + + _ = main(["--project-dir", str(project_dir), "compile", "--json", *args]) + return capsys.readouterr().out + + +def compile_outputs( + *, project_dir: Path, args: tuple[str, ...], capsys: pytest.CaptureFixture[str] +) -> CompileOutcome: + """Compile through the CLI; return the exit code, timing-free JSON and compiled files.""" + + exit_code: int = main(["--project-dir", str(project_dir), "compile", "--json", *args]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + for key in ("compile_timings", "compiler_engine"): + _ = payload.pop(key, None) + return exit_code, payload, compile_files(project_dir=project_dir) + + +def reuse_compile( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[bool, CompileOutcome]: + """Compile through the CLI with reuse on; return whether it replayed, and its outcome.""" + + exit_code: int = main(["--project-dir", str(project_dir), "compile", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + timings: dict[str, int] = cast(dict[str, int], payload.pop("compile_timings")) + _ = payload.pop("compiler_engine", None) + return timings["project_reuse_hits"] == 1, ( + exit_code, + payload, + compile_files(project_dir=project_dir), + ) + + +class ReuseComparison(NamedTuple): + """One reuse-enabled compile, whether it replayed, and an uncached compile of the project.""" + + reused: bool + outcome: tuple[int, dict[str, Any], dict[str, bytes]] + reference: tuple[int, dict[str, Any], dict[str, bytes]] + + +def reuse_against_uncached( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> ReuseComparison: + """Compile with reuse on, then compile a fresh copy with --no-cache, both comparable.""" + + reused, outcome = reuse_compile(project_dir=project_dir, capsys=capsys) + reference: CompileOutcome = uncached_reference( + project_dir=project_dir, reference_dir=project_dir.parent / "uncached", capsys=capsys + ) + return ReuseComparison( + reused=reused, outcome=comparable(outcome), reference=comparable(reference) + ) + + +def stored_compile_files(*, project_dir: Path) -> list[Path]: + """Every stored compile slot.""" + + return sorted( + (compiler_cache_directory(project_dir) / NATIVE_REUSE_DIRECTORY_NAME).glob( + f"*{NATIVE_REUSE_SUFFIX}" + ) + ) + + +def corrupt_stored_compiles(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Overwrite every stored compile file with bytes no reader accepts.""" + + del monkeypatch + for path in stored_compile_files(project_dir=project_dir): + _ = path.write_bytes(b"garbage") + + +def truncate_stored_compiles(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Cut every stored compile file in half.""" + + del monkeypatch + for path in stored_compile_files(project_dir=project_dir): + contents: bytes = path.read_bytes() + _ = path.write_bytes(contents[: len(contents) // 2]) + + +def change_runtime(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Report a different interpreter from now on.""" + + del project_dir + + def changed() -> dict[str, str]: + return {**runtime_identity(), "python": "3.0.0 (simulated)"} + + monkeypatch.setattr(native_reuse_module, "runtime_identity", changed) + + +def change_native_build(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Report another build of the native extension from now on, as after reinstalling it.""" + + del project_dir + monkeypatch.setattr(native_module, "BUILD_IDENTITY", f"{native_module.BUILD_IDENTITY}-rebuilt") + + +def edit_loaded_module(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Rewrite the imported helper module outside the project.""" + + del monkeypatch + module: Path = project_dir.parent / HELPER_MODULE_DIRECTORY / f"{HELPER_MODULE}.py" + _ = module.write_text(module.read_text(encoding="utf-8") + "# edited\n", encoding="utf-8") + + +def edit_then_fail_store(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Edit a model, then make every later store fail while enumerating provider settings.""" + + def raise_error(**_kwargs: object) -> None: + raise RuntimeError("settings went away") + + _edit(project_dir=project_dir) + monkeypatch.setattr(native_reuse_module, "provider_settings_inputs", raise_error) + + +def set_tracked_variable(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Set a SQLBuild environment variable no template reads.""" + + del project_dir + monkeypatch.setenv(TRACKED_VARIABLE, "north") + + +def set_untracked_variable(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Point stage captures somewhere; capture settings are not compile inputs.""" + + monkeypatch.setenv(STAGE_CAPTURE_DIR_ENV_VAR, str(project_dir.parent / "captures")) + + +def block_stored_compiles(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Replace every stored compile file with a non-empty directory.""" + + del monkeypatch + for path in stored_compile_files(project_dir=project_dir): + path.unlink() + path.mkdir() + _ = (path / "keep.txt").write_text("kept\n", encoding="utf-8") + + +def load_helper_module(*, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Import a helper module from outside the project, as an installed package would be.""" + + directory: Path = tmp_path / HELPER_MODULE_DIRECTORY + directory.mkdir() + _ = (directory / f"{HELPER_MODULE}.py").write_text("ORDERS = 1\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(directory)) + monkeypatch.delitem(sys.modules, HELPER_MODULE, raising=False) + _ = importlib.import_module(HELPER_MODULE) + + +def compile_files(*, project_dir: Path) -> dict[str, bytes]: + """Every file under target/compiled by its relative path.""" + + compiled: Path = project_dir / "target" / "compiled" + return { + path.relative_to(compiled).as_posix(): path.read_bytes() + for path in sorted(filter(Path.is_file, compiled.rglob("*"))) + } + + +def main_compile(*, project_dir: Path) -> int: + """Run one uncached JSON compile through the CLI entry point.""" + + return main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) + + +def uncached_reference( + *, project_dir: Path, reference_dir: Path, capsys: pytest.CaptureFixture[str] +) -> CompileOutcome: + """Compile a fresh copy of the project with --no-cache.""" + + shutil.rmtree(reference_dir, ignore_errors=True) + _ = shutil.copytree(project_dir, reference_dir, ignore=shutil.ignore_patterns("target")) + return compile_outputs(project_dir=reference_dir, args=("--no-cache",), capsys=capsys) + + +def comparable(outcome: CompileOutcome) -> tuple[int, dict[str, Any], dict[str, bytes]]: + """The outcome with project paths made relative, so two copies compare equal.""" + + exit_code, payload, files = outcome + artifacts: dict[str, Any] = cast(dict[str, Any], payload.get("artifacts", {})) + return exit_code, {**payload, "artifacts": sorted(artifacts)}, files + + +def work_delta(*, before: Counter[str], after: Counter[str]) -> dict[str, int]: + """Native compile output work counted between two snapshots, by kind.""" + + return { + kind.removeprefix(COMPILE_OUTPUTS_PREFIX): units + for kind, units in sorted((after - before).items()) + } + + +def staging_directories(target_dir: Path) -> list[str]: + """Staging directories left beside the compiled directory.""" + + return sorted(path.name for path in target_dir.glob(f"{STAGING_PREFIX}*")) + + +def block_artifact(*, project_dir: Path) -> Path: + """Replace one published artifact with a non-empty directory and edit its model.""" + + blocked: Path = project_dir / BLOCKED_ARTIFACT + blocked.unlink() + blocked.mkdir() + _ = (blocked / "keep.txt").write_text("kept\n", encoding="utf-8") + model: Path = project_dir / EDITED_MODEL + _ = model.write_text( + model.read_text(encoding="utf-8").replace("CAST(1 AS", "CAST(2 AS"), encoding="utf-8" + ) + return blocked + + +def neutral_error(*, error: BaseException, project_dir: Path) -> str: + """The error text with the project directory and staging name replaced.""" + + text: str = str(error).replace(str(project_dir), "") + return re.sub(rf"{re.escape(STAGING_PREFIX)}[0-9a-f]+", f"{STAGING_PREFIX}", text) + + +def edit_sequence_step(*, project_dir: Path, step: str) -> None: + """Apply one step of the cached edit sequence.""" + + _SEQUENCE_ACTIONS[step](project_dir=project_dir) + + +def _unchanged(*, project_dir: Path) -> None: + del project_dir + + +def _touch_back(*, project_dir: Path) -> None: + model: Path = project_dir / EDITED_MODEL + touched_ns: int = model.stat().st_mtime_ns - TOUCH_BACK_NS + os.utime(model, ns=(touched_ns, touched_ns)) + + +def _edit(*, project_dir: Path) -> None: + model: Path = project_dir / EDITED_MODEL + _ = model.write_text( + model.read_text(encoding="utf-8").replace("CAST(1 AS", "CAST(30 AS"), encoding="utf-8" + ) + + +def _delete(*, project_dir: Path) -> None: + (project_dir / DELETED_MODEL).unlink() + + +_SEQUENCE_ACTIONS: dict[str, Callable[..., None]] = { + "cold": _unchanged, + "warm": _unchanged, + "touch": _touch_back, + "retouch": _touch_back, + "edit": _edit, + "delete": _delete, +} + + +def published_files(counts: Counter[str]) -> int: + """Staged artifacts published natively.""" + + return counts[f"{COMPILE_OUTPUTS_PREFIX}published_files"] + + +def artifacts_except(*, files: dict[str, bytes], name: str) -> dict[str, bytes]: + """Every artifact except one.""" + + return {key: files[key] for key in sorted(files.keys() - {name})} diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_compile_outputs.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_compile_outputs.py new file mode 100644 index 0000000000..b56b6e8ae5 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_compile_outputs.py @@ -0,0 +1,154 @@ +"""Integration coverage for compile JSON, artifact writes, publication and reuse snapshots.""" + +import shutil +from collections import Counter +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.compile_outputs._test_types import ( + CompileOutputsSequenceTestCase, + PublicationFailureTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.compile_outputs.helpers import ( + OUTPUT_PROJECT_FILES, + CompileOutcome, + artifacts_except, + block_artifact, + comparable, + compile_files, + compile_outputs, + edit_sequence_step, + main_compile, + neutral_error, + published_files, + record_output_work, + staging_directories, + uncached_reference, + work_delta, + write_files, +) + +_SEQUENCE_STEPS: tuple[str, ...] = ("cold", "warm", "touch", "retouch", "edit", "delete") +_PUBLICATION_ERROR: str = ( + "[Errno 21] Is a directory: " + "'/target/.sqlbuild-staging-/compiled/models/orders_north.sql' -> " + "'/target/compiled/models/orders_north.sql'" +) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileOutputsSequenceTestCase( + description="compile writes outputs and reuses compiles natively", + expected_exit_codes=(0, 0, 0, 0, 0, 0), + expected_work=( + { + "artifact_files": 3, + "json_reports": 1, + "reuse_records": 1, + "reuse_snapshot_paths": 5, + }, + {"reuse_replays": 1, "reuse_snapshot_paths": 5}, + { + "artifact_files": 3, + "json_reports": 1, + "reuse_digested_files": 1, + "reuse_records": 1, + "reuse_snapshot_paths": 5, + }, + {"reuse_digested_files": 1, "reuse_replays": 1, "reuse_snapshot_paths": 5}, + { + "artifact_files": 3, + "json_reports": 1, + "reuse_digested_files": 1, + "reuse_records": 1, + "reuse_snapshot_paths": 5, + }, + { + "artifact_files": 2, + "json_reports": 1, + "reuse_records": 1, + "reuse_snapshot_paths": 4, + "stale_files_removed": 1, + }, + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_cached_edit_sequence_when_compiling_then_each_step_matches_uncached_compile( + test_case: CompileOutputsSequenceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_dir: Path = tmp_path / "project" + write_files(project_dir=project_dir, files=OUTPUT_PROJECT_FILES) + counts: Counter[str] = record_output_work(monkeypatch=monkeypatch, reuse_disabled="0") + work: list[dict[str, int]] = [] + exit_codes: list[int] = [] + + for step in _SEQUENCE_STEPS: + edit_sequence_step(project_dir=project_dir, step=step) + before: Counter[str] = Counter(counts) + cached: CompileOutcome = compile_outputs(project_dir=project_dir, args=(), capsys=capsys) + work.append(work_delta(before=before, after=counts)) + exit_codes.append(cached[0]) + reference: CompileOutcome = uncached_reference( + project_dir=project_dir, reference_dir=tmp_path / f"uncached_{step}", capsys=capsys + ) + assert comparable(cached) == comparable(reference) + + assert tuple(exit_codes) == test_case.expected_exit_codes + assert tuple(work) == test_case.expected_work + + +@pytest.mark.parametrize( + "test_case", + [ + PublicationFailureTestCase( + description="compile reports the blocked move and recovers", + expected_error=_PUBLICATION_ERROR, + expected_published_files=9, + ), + ], + ids=lambda case: case.description, +) +def test_given_artifact_path_blocked_when_publishing_staged_artifacts_then_compile_fails_cleanly( + test_case: PublicationFailureTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_files(project_dir=tmp_path, files=OUTPUT_PROJECT_FILES) + counts: Counter[str] = record_output_work(monkeypatch=monkeypatch, reuse_disabled="1") + published: CompileOutcome = compile_outputs( + project_dir=tmp_path, args=("--no-cache",), capsys=capsys + ) + blocked: Path = block_artifact(project_dir=tmp_path) + untouched: dict[str, bytes] = artifacts_except( + files=published[2], name="models/orders_north.sql" + ) + + with pytest.raises(IsADirectoryError) as raised: + _ = main_compile(project_dir=tmp_path) + remaining: dict[str, bytes] = compile_files(project_dir=tmp_path) + kept: str = (blocked / "keep.txt").read_text(encoding="utf-8") + shutil.rmtree(blocked) + recovered: CompileOutcome = compile_outputs( + project_dir=tmp_path, args=("--no-cache",), capsys=capsys + ) + reference: CompileOutcome = uncached_reference( + project_dir=tmp_path, + reference_dir=tmp_path.parent / f"{tmp_path.name}_uncached", + capsys=capsys, + ) + + assert neutral_error(error=raised.value, project_dir=tmp_path) == test_case.expected_error + assert staging_directories(tmp_path / "target") == [] + assert {name: remaining[name] for name in untouched} == untouched + assert kept == "kept\n" + assert comparable(recovered) == comparable(reference) + assert published_files(counts) == test_case.expected_published_files diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_compile_reuse.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_compile_reuse.py new file mode 100644 index 0000000000..43e6d25ea5 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_compile_reuse.py @@ -0,0 +1,113 @@ +"""Integration coverage for whole-project compile reuse when the stored compile is disrupted.""" + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main.compile_outputs._test_types import ( + ReuseDisruptionTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.compile_outputs.helpers import ( + OUTPUT_PROJECT_FILES, + ReuseComparison, + block_stored_compiles, + change_native_build, + change_runtime, + corrupt_stored_compiles, + edit_loaded_module, + edit_then_fail_store, + load_helper_module, + record_output_work, + reuse_against_uncached, + set_tracked_variable, + set_untracked_variable, + stored_compile_files, + truncate_stored_compiles, + write_files, +) + +_REPLAYED_AFTER: tuple[bool, ...] = (False, True, False, True) +_MISSED_AFTER: tuple[bool, ...] = (False, True, False, False) + + +@pytest.mark.parametrize( + "test_case", + [ + ReuseDisruptionTestCase( + description="corrupt stored compile", + disrupt=corrupt_stored_compiles, + expected_reused=_REPLAYED_AFTER, + expected_slots=1, + ), + ReuseDisruptionTestCase( + description="truncated stored compile", + disrupt=truncate_stored_compiles, + expected_reused=_REPLAYED_AFTER, + expected_slots=1, + ), + ReuseDisruptionTestCase( + description="changed interpreter", + disrupt=change_runtime, + expected_reused=_REPLAYED_AFTER, + expected_slots=1, + ), + ReuseDisruptionTestCase( + description="changed native build", + disrupt=change_native_build, + expected_reused=_REPLAYED_AFTER, + expected_slots=1, + ), + ReuseDisruptionTestCase( + description="edited loaded module", + disrupt=edit_loaded_module, + expected_reused=_REPLAYED_AFTER, + expected_slots=1, + ), + ReuseDisruptionTestCase( + description="unwritable stored compile", + disrupt=block_stored_compiles, + expected_reused=_MISSED_AFTER, + expected_slots=1, + ), + ReuseDisruptionTestCase( + description="failing store after an edit", + disrupt=edit_then_fail_store, + expected_reused=_MISSED_AFTER, + expected_slots=0, + ), + ReuseDisruptionTestCase( + description="tracked environment variable set", + disrupt=set_tracked_variable, + expected_reused=_REPLAYED_AFTER, + expected_slots=1, + ), + ReuseDisruptionTestCase( + description="untracked environment variable set", + disrupt=set_untracked_variable, + expected_reused=(False, True, True, True), + expected_slots=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_compile_disrupted_when_rerunning_then_it_misses_and_matches_uncached( + test_case: ReuseDisruptionTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_dir: Path = tmp_path / "project" + write_files(project_dir=project_dir, files=OUTPUT_PROJECT_FILES) + load_helper_module(tmp_path=tmp_path, monkeypatch=monkeypatch) + _ = record_output_work(monkeypatch=monkeypatch, reuse_disabled="0") + + cold: ReuseComparison = reuse_against_uncached(project_dir=project_dir, capsys=capsys) + warm: ReuseComparison = reuse_against_uncached(project_dir=project_dir, capsys=capsys) + test_case.disrupt(project_dir, monkeypatch) + disrupted: ReuseComparison = reuse_against_uncached(project_dir=project_dir, capsys=capsys) + after: ReuseComparison = reuse_against_uncached(project_dir=project_dir, capsys=capsys) + runs: tuple[ReuseComparison, ...] = (cold, warm, disrupted, after) + + assert [run.outcome for run in runs] == [run.reference for run in runs] + assert tuple(run.reused for run in runs) == test_case.expected_reused + assert len(stored_compile_files(project_dir=project_dir)) == test_case.expected_slots diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_json_report.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_json_report.py new file mode 100644 index 0000000000..75d90f9ccc --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_json_report.py @@ -0,0 +1,270 @@ +"""The native JSON report keeps the orjson report contract and its `json.dumps` fallback.""" + +import json +import logging +from collections import Counter +from collections.abc import Callable +from pathlib import Path + +import orjson +import pytest + +import sqlbuild._native as native_module +from sqlbuild.cli.commands._helpers.compile import output as output_module +from tests.integration.src.sqlbuild.cli.commands.main.compile_outputs._test_types import ( + CliJsonReportTestCase, + JsonReportErrorTestCase, + JsonReportTextTestCase, + NestedHookReportTestCase, + NestedReportTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.compile_outputs.helpers import ( + COMPILE_OUTPUTS_PREFIX, + SURROGATE_HOOK_FILES, + SURROGATE_SELECTOR, + compile_json_text, + main_compile_json, + nested_lists, + nested_payload_hook_files, + record_output_work, + write_files, +) + +CLI_LOGGER: str = "sqlbuild.cli" +_ORJSON_EMITTERS: dict[bool, Callable[[object], str | None]] = { + True: native_module.emit_orjson_report, + False: native_module.emit_json_report, +} + + +@pytest.mark.parametrize( + "test_case", + [ + JsonReportTextTestCase( + description="plain values are written as orjson writes them", + report={"name": "orders", "rows": [1, 2.5, None, True], "é": "é"}, + expected_text=orjson.dumps( + {"name": "orders", "rows": [1, 2.5, None, True], "é": "é"}, + option=orjson.OPT_INDENT_2, + ).decode(), + ), + JsonReportTextTestCase( + description="a non-finite float is null, as orjson writes it", + report={"ratio": float("nan")}, + expected_text=orjson.dumps( + {"ratio": float("nan")}, option=orjson.OPT_INDENT_2 + ).decode(), + ), + JsonReportTextTestCase( + description="the largest unsigned 64-bit integer stays with orjson", + report={"rows": 2**64 - 1}, + expected_text=orjson.dumps({"rows": 2**64 - 1}, option=orjson.OPT_INDENT_2).decode(), + ), + JsonReportTextTestCase( + description="a wider integer falls back to json.dumps for the whole report", + report={"rows": 2**70, "é": "é"}, + expected_text=json.dumps({"rows": 2**70, "é": "é"}, indent=2), + ), + JsonReportTextTestCase( + description="lone surrogates beside escapes, a non-BMP pair and a wide integer", + report={"text": '"q"\\é\udcff\U0001f600\ud83d', "rows": 2**70}, + expected_text=json.dumps( + {"text": '"q"\\é\udcff\U0001f600\ud83d', "rows": 2**70}, indent=2 + ), + ), + JsonReportTextTestCase( + description="253 nested lists under a key are the deepest orjson writes", + report={"a": nested_lists(depth=253)}, + expected_text=orjson.dumps( + {"a": nested_lists(depth=253)}, option=orjson.OPT_INDENT_2 + ).decode(), + ), + JsonReportTextTestCase( + description="orjson's recursion limit falls back to json.dumps for the whole report", + report={"a": nested_lists(depth=254)}, + expected_text=json.dumps({"a": nested_lists(depth=254)}, indent=2), + ), + JsonReportTextTestCase( + description="scalar keys fall back to json.dumps key text", + report={1: "a", 1.5: "b", float("nan"): "c", None: "d", False: "e"}, + expected_text=json.dumps( + {1: "a", 1.5: "b", float("nan"): "c", None: "d", False: "e"}, indent=2 + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_report_values_when_emitting_natively_then_text_matches_the_shipped_encoders( + test_case: JsonReportTextTestCase, +) -> None: + assert native_module.emit_json_report(test_case.report) == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + [ + JsonReportErrorTestCase( + description="error report: lone surrogate", + report={"message": "unknown selector 'x\udcff'"}, + orjson_only=True, + expected_message="str is not valid UTF-8: surrogates not allowed", + ), + JsonReportErrorTestCase( + description="error report: integer beyond 64 bits", + report={"line": -(2**64)}, + orjson_only=True, + expected_message="Integer exceeds 64-bit range", + ), + JsonReportErrorTestCase( + description="error report: orjson's recursion limit", + report={"a": nested_lists(depth=254)}, + orjson_only=True, + expected_message="Recursion limit reached", + ), + JsonReportErrorTestCase( + description="error report: integer key", + report={1: "orders"}, + orjson_only=True, + expected_message="Dict key must be str", + ), + JsonReportErrorTestCase( + description="error report: value neither encoder writes", + report={"names": frozenset()}, + orjson_only=True, + expected_message="Type is not JSON serializable: frozenset", + ), + JsonReportErrorTestCase( + description="full report: value neither encoder writes", + report={"names": frozenset()}, + orjson_only=False, + expected_message="Object of type frozenset is not JSON serializable", + ), + JsonReportErrorTestCase( + description="full report: key neither encoder writes", + report={frozenset(): "orders"}, + orjson_only=False, + expected_message="keys must be str, int, float, bool or None, not frozenset", + ), + ], + ids=lambda case: case.description, +) +def test_given_report_value_the_shipped_encoder_rejects_when_emitting_then_same_type_error( + test_case: JsonReportErrorTestCase, +) -> None: + with pytest.raises(TypeError) as raised: + _ = _ORJSON_EMITTERS[test_case.orjson_only](test_case.report) + + assert str(raised.value) == test_case.expected_message + + +@pytest.mark.parametrize( + "test_case", + [ + CliJsonReportTestCase( + description="hook docstring with a lone surrogate", + args=("--no-cache",), + expected_fragment=( + r'"description": "Notify \"q\" \\ caf\u00e9 ' + r'\udcff\ud83d\ude00\ud83d\ude00\ud83d end."' + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_lone_surrogate_in_full_report_when_compiling_json_then_native_writes_json_dumps_bytes( + test_case: CliJsonReportTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_files(project_dir=tmp_path, files=SURROGATE_HOOK_FILES) + counts: Counter[str] = record_output_work(monkeypatch=monkeypatch, reuse_disabled="1") + + text: str = compile_json_text(project_dir=tmp_path, args=test_case.args, capsys=capsys) + + assert test_case.expected_fragment in text + assert text.isascii() + assert text == json.dumps(json.loads(text), indent=2) + "\n" + assert counts[f"{COMPILE_OUTPUTS_PREFIX}json_reports"] == 1 + assert not hasattr(output_module, "json") + assert not hasattr(output_module, "orjson") + + +@pytest.mark.parametrize( + "test_case", + [ + CliJsonReportTestCase( + description="selector with undecodable bytes", + args=("--no-cache", "--select", SURROGATE_SELECTOR), + expected_fragment="str is not valid UTF-8: surrogates not allowed", + ) + ], + ids=lambda case: case.description, +) +def test_given_lone_surrogate_in_error_report_when_compiling_json_then_orjson_error_propagates( + test_case: CliJsonReportTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + caplog.set_level(logging.CRITICAL, logger=CLI_LOGGER) + write_files(project_dir=tmp_path, files=SURROGATE_HOOK_FILES) + _ = record_output_work(monkeypatch=monkeypatch, reuse_disabled="1") + + with pytest.raises(TypeError) as raised: + _ = compile_json_text(project_dir=tmp_path, args=test_case.args, capsys=capsys) + _ = capsys.readouterr() + + assert str(raised.value) == test_case.expected_fragment + + +@pytest.mark.parametrize( + "test_case", + [ + NestedHookReportTestCase( + description="hook payload 122 lists deep inside the report wrappers", + depth=122, + expected_exit_code=0, + ) + ], + ids=lambda case: case.description, +) +def test_given_deeply_nested_hook_payload_when_compiling_json_then_orjson_writes_the_report( + test_case: NestedHookReportTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_files(project_dir=tmp_path, files=nested_payload_hook_files(depth=test_case.depth)) + counts: Counter[str] = record_output_work(monkeypatch=monkeypatch, reuse_disabled="1") + + exit_code: int = main_compile_json(project_dir=tmp_path) + text: str = capsys.readouterr().out + + assert exit_code == test_case.expected_exit_code + assert text == orjson.dumps(json.loads(text), option=orjson.OPT_INDENT_2).decode() + "\n" + assert '"payload": [' in text + assert counts[f"{COMPILE_OUTPUTS_PREFIX}json_reports"] == 1 + + +@pytest.mark.parametrize( + "test_case", + [ + NestedReportTestCase( + description="beyond Python's default recursion limit", + depth=1_000, + expected_error=RecursionError, + ) + ], + ids=lambda case: case.description, +) +def test_given_report_deeper_than_python_recursion_when_emitting_then_recursion_error( + test_case: NestedReportTestCase, +) -> None: + with pytest.raises(test_case.expected_error): + _ = native_module.emit_json_report({"a": nested_lists(depth=test_case.depth)}) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index e5910cb01d..a9a33dbdd5 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -5,6 +5,7 @@ import sys import tempfile import threading +from collections import Counter from collections.abc import Callable from pathlib import Path from types import ModuleType @@ -29,13 +30,16 @@ from sqlbuild.cli.compile.models import PlannedStaticSqlTests from sqlbuild.compiler.analysis_session.models import NativeQueryColumns from sqlbuild.compiler.compile._helpers.assembly import source_bindings as source_bindings_module +from sqlbuild.compiler.compile.main import expand_sql_with_spans as expand_sql_module from sqlbuild.compiler.discovery._helpers.native import ( model_files as native_model_files_module, ) +from sqlbuild.compiler.frontier.types import NativeFallbackSite, NativeStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from sqlbuild.lint._helpers import fixes as lint_fixes from sqlbuild.lint.main import run_format as run_format_module +from sqlbuild.rule_engine._helpers.run import rules as rules_run_module from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, ) @@ -826,3 +830,62 @@ def write_default_connection_sql_test(*, project_dir: Path) -> None: "SELECT 1\n", encoding="utf-8", ) + + +def write_project_files(*, project_dir: Path, files: dict[str, str]) -> None: + """Write project files at their project-relative paths.""" + + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + +EARLY_LINT_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nfloor = "2"\n\n' + '[rules]\nselect = ["SQBRSQL005"]\n' + ), + "models/_macros/cents.py": ( + 'def cents(expression: str) -> str:\n """Convert to cents."""\n' + ' return f"({expression} * 100)"\n' + ), + "models/orders.sql": ( + 'MODEL (description "Orders");\nSELECT 1 AS order_id, CAST(5 AS DOUBLE) AS amount\n' + ), + "models/order_cents.sql": ( + 'MODEL (description "Order amounts in cents");\n' + 'WITH floored AS (SELECT order_id, @cents("amount") AS amount_cents, ' + '@@floor AS floor_amount FROM __ref("orders")), spare AS (SELECT 1 AS one)\n' + "SELECT order_id, amount_cents FROM floored WHERE amount_cents > floor_amount\n" + ), +} + + +def record_early_lint_expansions(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: + """Count model expansions early lint took from the compile and expanded again in Python.""" + + counts: Counter[str] = Counter() + + def handed_over(*, stage: NativeStage, kind: str, units: int = 1) -> None: + counts[f"{stage.value}:{kind}"] += units + + def expanded_again(*, site: NativeFallbackSite, kind: str = "deferred") -> None: + counts[f"{site.value}:{kind}"] += 1 + + monkeypatch.setattr(rules_run_module, "report_native_answer", handed_over) + monkeypatch.setattr(expand_sql_module, "report_native_fallback", expanded_again) + return counts + + +def compile_findings( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[tuple[str, str, int | None, int | None], ...]: + """Compile through the CLI with no cache and return each finding's code and location.""" + + _ = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) + payload: dict[str, object] = json.loads(capsys.readouterr().out) + diagnostics: list[dict[str, Any]] = cast(list[dict[str, Any]], payload["diagnostics"]) + return tuple( + (item["code"], item["path"], item.get("line"), item.get("column")) for item in diagnostics + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_default_connection.py b/tests/integration/src/sqlbuild/cli/commands/main/test_default_connection.py index dc753f24b6..00c8819229 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_default_connection.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_default_connection.py @@ -11,6 +11,7 @@ from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( DefaultConnectionCliTestCase, InferredFreshnessCliTestCase, + LocalOverrideCliTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( execute_duckdb_sql, @@ -42,6 +43,20 @@ _NO_CONNECTION_PROJECT: str = ( 'name = "shop"\nadapter = "duckdb"\ndefault_target = "dev"\n\n[targets.dev]\nschema = "dev"\n' ) +_TEMPLATED_CONNECTION_PROJECT: str = ( + 'name = "shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[vars]\n" + 'db_name = "shop"\n\n' + "[connections.local]\n" + 'database = "${db_name}.duckdb"\n\n' + "[connections.archive]\n" + 'database = "archive.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' + 'schema = "dev"\n' +) _MEMORY_CONNECTION_PROJECT: str = ( 'name = "shop"\n' 'adapter = "duckdb"\n' @@ -85,6 +100,54 @@ def test_given_target_without_connection_when_building_then_only_named_connectio ) +@pytest.mark.parametrize( + "test_case", + [ + LocalOverrideCliTestCase( + description="local vars rename the templated database and local schema wins", + local_toml='[vars]\ndb_name = "mine"\n\n[targets.dev]\nschema = "staging"\n', + argv=("build",), + expected_exit_code=0, + expected_database="mine.duckdb", + expected_schema="staging", + expected_order_rows=((1,),), + ), + LocalOverrideCliTestCase( + description="CLI vars beat local vars and a local target switches connection", + local_toml='[vars]\ndb_name = "mine"\n\n[targets.dev]\nconnection = "archive"\n', + argv=("build", "--vars", '{"db_name": "cli"}'), + expected_exit_code=0, + expected_database="archive.duckdb", + expected_schema="dev", + expected_order_rows=((1,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_local_and_cli_overrides_when_building_then_precedence_picks_the_relation( + test_case: LocalOverrideCliTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_default_connection_project( + project_dir=tmp_path, project_toml=_TEMPLATED_CONNECTION_PROJECT + ) + _ = (tmp_path / "sqlbuild_local.toml").write_text(test_case.local_toml, encoding="utf-8") + + exit_code: int = main(["--no-color", "--project-dir", str(tmp_path), *test_case.argv]) + + output: str = "".join(capsys.readouterr()) + assert exit_code == test_case.expected_exit_code, output + assert ( + query_duckdb_rows( + db_path=tmp_path / test_case.expected_database, + sql=f"SELECT order_id FROM {test_case.expected_schema}.orders", + ) + == test_case.expected_order_rows + ) + assert not (tmp_path / "shop.duckdb").exists() + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d449de76ef..23734e623c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from collections import Counter from collections.abc import Callable from pathlib import Path from threading import Event @@ -27,6 +28,7 @@ from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( CustomHostSplitIntegrationTestCase, DynamicPivotRulesIntegrationTestCase, + EarlyLintInputsTestCase, ExplicitContractOutputRuleIntegrationTestCase, ImplicitAliasRuleIntegrationTestCase, NumericRangeDecisionIntegrationTestCase, @@ -34,9 +36,18 @@ RulesIntegrationTestCase, TypedContractRuleIntegrationTestCase, ) -from tests.integration.src.sqlbuild.cli.commands.main.helpers import compile_finding_keys +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + EARLY_LINT_PROJECT_FILES, + compile_finding_keys, + compile_findings, + record_early_lint_expansions, + write_project_files, +) _EARLY_LINT_STOP_TIMEOUT_S: float = 30.0 +_EARLY_LINT_FINDINGS: tuple[tuple[str, str, int | None, int | None], ...] = ( + ("SQBRSQL005", "models/order_cents.sql", 2, 116), +) @pytest.mark.parametrize( @@ -934,6 +945,76 @@ def counting_collect( assert collections == [None] +@pytest.mark.parametrize( + "test_case", + [ + EarlyLintInputsTestCase( + description="compile lints every model from the compile's expansion", + expected_findings=_EARLY_LINT_FINDINGS, + expected_compile_expansions=2, + expected_python_expansions=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_variable_and_macro_models_when_compiling_then_early_lint_findings_match( + test_case: EarlyLintInputsTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_project_files(project_dir=tmp_path, files=EARLY_LINT_PROJECT_FILES) + counts: Counter[str] = record_early_lint_expansions(monkeypatch=monkeypatch) + + findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( + project_dir=tmp_path, capsys=capsys + ) + + assert findings == test_case.expected_findings + assert ( + counts["compile_lint_inputs:lint_expansions"], + counts["model_loop.lint_expansion:deferred"], + ) == (test_case.expected_compile_expansions, test_case.expected_python_expansions) + + +@pytest.mark.parametrize( + "test_case", + [ + EarlyLintInputsTestCase( + description="old build entries leave early lint enabled", + expected_findings=_EARLY_LINT_FINDINGS, + expected_compile_expansions=2, + expected_python_expansions=0, + ) + ], + ids=lambda case: case.description, +) +def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_same_findings( + test_case: EarlyLintInputsTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_project_files(project_dir=tmp_path, files=EARLY_LINT_PROJECT_FILES) + with monkeypatch.context() as previous_build: + previous_build.setattr(rules_module, "_RULES_BUILD_IDENTITY", "previous-build") + previous_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( + project_dir=tmp_path, capsys=capsys + ) + counts: Counter[str] = record_early_lint_expansions(monkeypatch=monkeypatch) + + upgraded_findings: tuple[tuple[str, str, int | None, int | None], ...] = compile_findings( + project_dir=tmp_path, capsys=capsys + ) + + assert previous_findings == upgraded_findings == test_case.expected_findings + assert counts["compile_lint_inputs:lint_expansions"] == test_case.expected_compile_expansions + assert counts["model_loop.lint_expansion:deferred"] == test_case.expected_python_expansions + counts.clear() + assert compile_findings(project_dir=tmp_path, capsys=capsys) == upgraded_findings + assert counts["compile_lint_inputs:lint_expansions"] == 0 + + @pytest.mark.parametrize( "test_case", [RulePassIntegrationTestCase("graph error stops the unused early lint", 1, ("B002",))], diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/__init__.py b/tests/integration/src/sqlbuild/cli/entry/__init__.py similarity index 100% rename from tests/unit/src/sqlbuild/compiler/refactoring/__init__.py rename to tests/integration/src/sqlbuild/cli/entry/__init__.py diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py new file mode 100644 index 0000000000..a0bad44aca --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -0,0 +1,56 @@ +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class HandoffTestCase: + description: str + args: tuple[str, ...] + expected_exit_code: int + + +@dataclass(frozen=True) +class TextReuseTestCase: + description: str + args: tuple[str, ...] + expected_notice: bytes + + +@dataclass(frozen=True) +class JsonReuseTestCase: + description: str + args: tuple[str, ...] + expected_hits: int + + +@dataclass(frozen=True) +class EditedProjectTestCase: + description: str + args: tuple[str, ...] + old_text: str + new_text: str + expected_misses: int + expected_fragment: str + + +@dataclass(frozen=True) +class SelectorFileTestCase: + description: str + selectors: str + expected_records: int + + +@dataclass(frozen=True) +class PythonStartsTestCase: + description: str + args: tuple[str, ...] + expected_python_starts: tuple[int, int, int] + + +@dataclass(frozen=True) +class ReuseFileMutationTestCase: + description: str + prepare: Callable[[Path], Path] + expected_misses: int + expected_fragment: str diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py new file mode 100644 index 0000000000..cfb0c8ed1b --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -0,0 +1,186 @@ +import json +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path + +import sqlbuild +from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR + +type LauncherRun = tuple[int, bytes, bytes] + +_ELAPSED_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d s\)") +NATIVE_SQB: Path = Path(sys.executable).with_name("sqb") +PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") +PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders_shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "warehouse.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' + 'schema = "dev"\n' + ), + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, CAST('2026-04-01' AS DATE) AS ordered_on\n" + ), +} + + +def write_project(project_dir: Path) -> Path: + """Write a one-model DuckDB project.""" + + for relative, contents in PROJECT_FILES.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + return project_dir + + +def run(program: Path, *, args: tuple[str, ...], cwd: Path) -> LauncherRun: + """Run one CLI process with reuse enabled; return its exit code, stdout and stderr.""" + + completed: subprocess.CompletedProcess[bytes] = subprocess.run( + [str(program), *args], + cwd=cwd, + capture_output=True, + check=False, + env={**os.environ, "NO_COLOR": "1", REUSE_DISABLE_ENV_VAR: "0"}, + timeout=600, + ) + return completed.returncode, completed.stdout, completed.stderr + + +def launch_records(project_dir: Path) -> dict[Path, int]: + """Every launch record in the project and its modification time.""" + + directory: Path = project_dir / "target" / "cache" / "launcher-v1" + return {path: path.stat().st_mtime_ns for path in directory.glob("*.json")} + + +def compute_log_captures(project_dir: Path) -> set[Path]: + """Every completed compute log capture directory.""" + + return {marker.parent for marker in (project_dir / "logs").glob("*/*/complete")} + + +def capture_metadata(capture: Path) -> dict[str, object]: + """A capture's final metadata.""" + + return json.loads((capture / "metadata.json").read_text(encoding="utf-8")) + + +def without_seconds(stream: bytes) -> bytes: + """The stream with the reuse notice's elapsed seconds blanked.""" + + return _ELAPSED_SECONDS.sub(b"(s)", stream) + + +def compile_timings(stdout: bytes) -> dict[str, int]: + """The `compile_timings` of a JSON compile report.""" + + return json.loads(stdout)["compile_timings"] + + +def without_compile_timings(stdout: bytes) -> dict[str, object]: + """Compare full reports while allowing invocation timings to differ.""" + + payload: dict[str, object] = json.loads(stdout) + _ = payload.pop("compile_timings") + return payload + + +def counting_install(directory: Path, marker: Path) -> Path: + """Install `sqb` beside a `sqlbuild-python` that logs each start to `marker`; return `sqb`.""" + + directory.mkdir(parents=True) + program: Path = Path(shutil.copy2(NATIVE_SQB, directory / NATIVE_SQB.name)) + script: Path = directory / PYTHON_CLI.name + _ = script.write_text( + f'#!/bin/sh\necho started >> "{marker}"\nexec "{PYTHON_CLI}" "$@"\n', encoding="utf-8" + ) + script.chmod(0o755) + return program + + +def python_starts(marker: Path) -> int: + """How many times the logging `sqlbuild-python` started.""" + + return len(marker.read_text(encoding="utf-8").splitlines()) + + +def write_linked_project(project_dir: Path) -> Path: + """Put the model behind a file link to an external input.""" + + project: Path = write_project(project_dir) + model: Path = project / "models" / "orders.sql" + target: Path = project.parent / "linked_orders.sql" + _ = model.replace(target) + model.symlink_to(target) + return project + + +def rewrite_model_preserving_mtime(project_dir: Path) -> None: + """Change equal-length SQL bytes, including through a link, without changing mtime.""" + + model: Path = project_dir / "models" / "orders.sql" + original: os.stat_result = model.stat() + _ = model.write_text(model.read_text().replace("1 AS order_id", "2 AS order_id")) + os.utime(model, ns=(original.st_atime_ns, original.st_mtime_ns)) + + +def run_with_full_output(program: Path, *, args: tuple[str, ...], cwd: Path) -> LauncherRun: + """Run with a writable descriptor that rejects every output byte.""" + + with Path("/dev/full").open("wb") as output: + completed: subprocess.CompletedProcess[bytes] = subprocess.run( + [str(program), *args], + cwd=cwd, + stdout=output, + stderr=subprocess.PIPE, + env={**os.environ, "NO_COLOR": "1", REUSE_DISABLE_ENV_VAR: "0"}, + check=False, + timeout=600, + ) + return completed.returncode, b"", completed.stderr + + +def startup_install(directory: Path) -> tuple[Path, Path, Path]: + """Install a launcher in an isolated interpreter with an editable startup search path.""" + + subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", "--system-site-packages", str(directory)], + check=True, + capture_output=True, + ) + scripts: Path = directory / "bin" + program: Path = Path(shutil.copy2(NATIVE_SQB, scripts / "sqb")) + handoff: Path = scripts / "sqlbuild-python" + _ = handoff.write_text( + f"#!{scripts / 'python'}\n" + "from sqlbuild.cli.entry.main.launched import main\nraise SystemExit(main())\n", + encoding="utf-8", + ) + handoff.chmod(0o755) + site_packages: Path = next((directory / "lib").glob("python*/site-packages")) + _ = (site_packages / "test_dependencies.pth").write_text( + str(Path(sqlbuild.__file__).resolve().parent.parent) + + "\n" + + "\n".join(str(Path(entry).resolve()) for entry in sys.path) + + "\n", + encoding="utf-8", + ) + old: Path = directory.parent / "old_helpers" + new: Path = directory.parent / "new_helpers" + old.mkdir() + new.mkdir() + _ = (old / "catalog_helper.py").write_text("SQL = '1'\n", encoding="utf-8") + _ = (new / "catalog_helper.py").write_text("SQL = '2'\n", encoding="utf-8") + startup: Path = site_packages / "catalog.pth" + _ = startup.write_text(str(old) + "\n", encoding="utf-8") + return program, startup, new diff --git a/tests/integration/src/sqlbuild/cli/entry/test_native_launcher.py b/tests/integration/src/sqlbuild/cli/entry/test_native_launcher.py new file mode 100644 index 0000000000..2cb019c792 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/test_native_launcher.py @@ -0,0 +1,313 @@ +import sys +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.entry._test_types import ( + EditedProjectTestCase, + HandoffTestCase, + JsonReuseTestCase, + PythonStartsTestCase, + ReuseFileMutationTestCase, + SelectorFileTestCase, + TextReuseTestCase, +) +from tests.integration.src.sqlbuild.cli.entry.helpers import ( + NATIVE_SQB, + PYTHON_CLI, + LauncherRun, + capture_metadata, + compile_timings, + compute_log_captures, + counting_install, + launch_records, + python_starts, + rewrite_model_preserving_mtime, + run, + run_with_full_output, + startup_install, + without_compile_timings, + without_seconds, + write_linked_project, + write_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + HandoffTestCase(description="version", args=("--version",), expected_exit_code=0), + HandoffTestCase(description="help", args=("--help",), expected_exit_code=0), + HandoffTestCase( + description="unknown command", args=("not-a-command",), expected_exit_code=2 + ), + HandoffTestCase( + description="compile outside a project", + args=("compile", "--json"), + expected_exit_code=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_any_invocation_when_running_native_sqb_then_streams_and_exit_match_python( + test_case: HandoffTestCase, tmp_path: Path +) -> None: + native: LauncherRun = run(NATIVE_SQB, args=test_case.args, cwd=tmp_path) + python: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=tmp_path) + + assert native == python + assert native[0] == test_case.expected_exit_code + + +@pytest.mark.parametrize( + "test_case", + [ + TextReuseTestCase( + description="text report", + args=("compile",), + expected_notice=b"Inputs unchanged; reused the previous compile (s)\n", + ) + ], + ids=lambda case: case.description, +) +def test_given_unchanged_project_when_compiling_again_then_sqb_replays_without_python( + test_case: TextReuseTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "project") + _ = run(NATIVE_SQB, args=test_case.args, cwd=project_dir) + records: dict[Path, int] = launch_records(project_dir) + captures: set[Path] = compute_log_captures(project_dir) + + served: LauncherRun = run(NATIVE_SQB, args=test_case.args, cwd=project_dir) + (served_capture,) = compute_log_captures(project_dir) - captures + python_hit: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=project_dir) + + assert len(records) == 1 + assert launch_records(project_dir) == records, "a Python hit rewrites its launch record" + assert served[0] == python_hit[0] == 0 + assert served[1] == python_hit[1] + assert without_seconds(served[2]) == without_seconds(python_hit[2]) + assert without_seconds(served[2]).startswith(test_case.expected_notice) + assert (served_capture / "stdout.log").read_bytes() == served[1] + assert (served_capture / "stderr.log").read_bytes() == served[2] + assert capture_metadata(served_capture) | {"started_at": None, "completed_at": None} == { + **capture_metadata(served_capture), + "started_at": None, + "completed_at": None, + "command": "compile", + "exit_code": 0, + "project_dir": str(project_dir.resolve()), + "stdout_bytes": len(served[1]), + "stderr_bytes": len(served[2]), + "diagnostics_bytes": 0, + "target": None, + "run_id": None, + } + + +@pytest.mark.parametrize( + "test_case", + [JsonReuseTestCase(description="json report", args=("compile", "--json"), expected_hits=1)], + ids=lambda case: case.description, +) +def test_given_unchanged_project_when_compiling_json_again_then_sqb_reports_a_reuse_hit( + test_case: JsonReuseTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "project") + _ = run(NATIVE_SQB, args=test_case.args, cwd=project_dir) + records: dict[Path, int] = launch_records(project_dir) + + served: LauncherRun = run(NATIVE_SQB, args=test_case.args, cwd=project_dir) + python_hit: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=project_dir) + + assert launch_records(project_dir) == records + assert compile_timings(served[1])["project_reuse_hits"] == test_case.expected_hits + assert compile_timings(served[1]).keys() == compile_timings(python_hit[1]).keys() + assert served[1].replace(b" ", b"").count(b'_ms":') == python_hit[1].replace(b" ", b"").count( + b'_ms":' + ) + + +@pytest.mark.parametrize( + "test_case", + [ + EditedProjectTestCase( + description="added column", + args=("compile", "--json"), + old_text="1 AS order_id", + new_text="2 AS order_id, 3 AS units", + expected_misses=1, + expected_fragment="units", + ) + ], + ids=lambda case: case.description, +) +def test_given_edited_model_when_compiling_through_sqb_then_python_recompiles_the_change( + test_case: EditedProjectTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "project") + _ = run(NATIVE_SQB, args=test_case.args, cwd=project_dir) + model: Path = project_dir / "models" / "orders.sql" + _ = model.write_text( + model.read_text(encoding="utf-8").replace(test_case.old_text, test_case.new_text), + encoding="utf-8", + ) + + edited: LauncherRun = run(NATIVE_SQB, args=test_case.args, cwd=project_dir) + + assert edited[0] == 0 + assert compile_timings(edited[1])["project_reuse_misses"] == test_case.expected_misses + assert test_case.expected_fragment in edited[1].decode() + + +@pytest.mark.parametrize( + "test_case", + [SelectorFileTestCase(description="orders", selectors="orders\n", expected_records=0)], + ids=lambda case: case.description, +) +def test_given_selector_file_when_compiling_through_sqb_then_no_launch_record_is_left( + test_case: SelectorFileTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "project") + selectors: Path = tmp_path / "selectors.txt" + _ = selectors.write_text(test_case.selectors, encoding="utf-8") + args: tuple[str, ...] = ("compile", "--select-file", str(selectors)) + + first: LauncherRun = run(NATIVE_SQB, args=args, cwd=project_dir) + second: LauncherRun = run(NATIVE_SQB, args=args, cwd=project_dir) + + assert first[0] == second[0] == 0 + assert len(launch_records(project_dir)) == test_case.expected_records + + +@pytest.mark.skipif(sys.platform == "win32", reason="the logging handoff is a POSIX shell script") +@pytest.mark.parametrize( + "test_case", + [ + PythonStartsTestCase( + description="text compile", + args=("compile",), + expected_python_starts=(1, 1, 2), + ), + PythonStartsTestCase( + description="json compile", + args=("compile", "--json"), + expected_python_starts=(1, 1, 2), + ), + ], + ids=lambda case: case.description, +) +def test_given_unchanged_project_when_sqb_serves_the_hit_then_python_never_starts( + test_case: PythonStartsTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "project") + marker: Path = tmp_path / "python-starts.log" + program: Path = counting_install(tmp_path / "bin", marker) + + first: LauncherRun = run(program, args=test_case.args, cwd=project_dir) + after_first: int = python_starts(marker) + served: LauncherRun = run(program, args=test_case.args, cwd=project_dir) + after_served: int = python_starts(marker) + model: Path = project_dir / "models" / "orders.sql" + _ = model.write_text( + model.read_text(encoding="utf-8").replace("1 AS order_id", "2 AS order_id"), + encoding="utf-8", + ) + edited: LauncherRun = run(program, args=test_case.args, cwd=project_dir) + + assert (first[0], served[0], edited[0]) == (0, 0, 0) + assert without_seconds(served[2]).startswith(b"Inputs unchanged; reused the previous compile") + assert (after_first, after_served, python_starts(marker)) == test_case.expected_python_starts + + +@pytest.mark.skipif(sys.platform == "win32", reason="file-link fixture requires POSIX symlinks") +@pytest.mark.parametrize( + "test_case", + [ + ReuseFileMutationTestCase( + "equal-length model rewrite preserving mtime", write_project, 1, "2 AS order_id" + ), + ReuseFileMutationTestCase( + "linked model target rewrite preserving mtime", write_linked_project, 1, "2 AS order_id" + ), + ], + ids=lambda case: case.description, +) +def test_given_cached_model_when_same_size_input_changes_then_replay_matches_fresh_compile( + test_case: ReuseFileMutationTestCase, + tmp_path: Path, +) -> None: + project: Path = test_case.prepare(tmp_path / "project") + args: tuple[str, ...] = ("compile", "--json") + first: LauncherRun = run(NATIVE_SQB, args=args, cwd=project) + assert first[0] == 0, first[2] + warm: LauncherRun = run(NATIVE_SQB, args=args, cwd=project) + assert warm[0] == 0, warm[2] + assert compile_timings(warm[1])["project_reuse_hits"] == 1 + rewrite_model_preserving_mtime(project) + edited: LauncherRun = run(NATIVE_SQB, args=args, cwd=project) + fresh: LauncherRun = run(PYTHON_CLI, args=(*args, "--no-cache"), cwd=project) + assert edited[0] == fresh[0] == 0 + assert compile_timings(edited[1])["project_reuse_misses"] == test_case.expected_misses + assert test_case.expected_fragment.encode() in edited[1] + assert without_compile_timings(edited[1]) == without_compile_timings(fresh[1]) + + +@pytest.mark.skipif(sys.platform != "linux", reason="requires /dev/full") +@pytest.mark.parametrize( + "test_case", + [HandoffTestCase("failed replay output", ("compile", "--json"), 1)], + ids=lambda case: case.description, +) +def test_given_cached_compile_when_output_write_fails_then_replay_returns_failure( + test_case: HandoffTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = write_project(tmp_path / "project") + first: LauncherRun = run(NATIVE_SQB, args=test_case.args, cwd=project_dir) + assert first[0] == 0 + assert launch_records(project_dir) + captures: set[Path] = compute_log_captures(project_dir) + failed: LauncherRun = run_with_full_output(NATIVE_SQB, args=test_case.args, cwd=project_dir) + assert failed[0] == test_case.expected_exit_code + (capture,) = compute_log_captures(project_dir) - captures + assert capture_metadata(capture)["exit_code"] == test_case.expected_exit_code + + +@pytest.mark.skipif(sys.platform == "win32", reason="isolated install uses POSIX scripts") +@pytest.mark.parametrize( + "test_case", + [ + EditedProjectTestCase( + "startup search path edit", ("compile", "--json"), "1", "2", 1, "2 AS order_id" + ) + ], + ids=lambda case: case.description, +) +def test_given_cached_compile_when_startup_search_path_changes_then_new_helper_is_compiled( + test_case: EditedProjectTestCase, + tmp_path: Path, +) -> None: + program, startup, new_helpers = startup_install(tmp_path / "environment") + project_dir: Path = write_project(tmp_path / "project") + (project_dir / "models" / "_macros").mkdir() + _ = (project_dir / "models" / "_macros" / "orders.py").write_text( + 'import catalog_helper\n\ndef order_value() -> str:\n """The order value."""\n return catalog_helper.SQL\n', + encoding="utf-8", + ) + model: Path = project_dir / "models" / "orders.sql" + _ = model.write_text(model.read_text().replace("1 AS order_id", "@order_value() AS order_id")) + first: LauncherRun = run(program, args=test_case.args, cwd=project_dir) + assert first[0] == 0, first[2] + assert launch_records(project_dir) + _ = run(program, args=test_case.args, cwd=project_dir) + warm: LauncherRun = run(program, args=test_case.args, cwd=project_dir) + assert compile_timings(warm[1])["project_reuse_hits"] == 1 + _ = startup.write_text(str(new_helpers) + "\n", encoding="utf-8") + edited: LauncherRun = run(program, args=test_case.args, cwd=project_dir) + assert edited[0] == 0, edited[2] + assert compile_timings(edited[1])["project_reuse_misses"] == test_case.expected_misses + assert ( + test_case.expected_fragment + in (project_dir / "target" / "compiled" / "models" / "orders.sql").read_text() + ) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 9f149afe19..488dfcfe34 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -15,26 +15,33 @@ import pytest import sqlbuild._native as native_module -import sqlbuild.compiler.analysis_session.classes.native_model_analysis as native_model_analysis import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main +from sqlbuild.compiler.analysis_session.constants import LINEAGE_NATIVE from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( analyze_native_model_sql, ) +from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import ( + dynamic_contract_proof as contract_proof, +) from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( infer_native_expression_source_shapes, ) from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contracts import ( prove_native_dynamic_contracts, ) +from sqlbuild.compiler.analysis_session.main._session_outcome_analysis import ( + session_outcome_analysis, +) from sqlbuild.compiler.analysis_session.models import ( NativeModelAnalyses, NativeModelAnalysisRequest, NativePivotTables, ) +from sqlbuild.compiler.analysis_session.types import ProofRow from sqlbuild.compiler.compile._helpers.analysis.cache import build_analysis_cache_context from sqlbuild.compiler.compile._helpers.analysis.pivot_requests import ( model_dynamic_families, @@ -51,14 +58,11 @@ AnalysisCacheContext, CompactLineageFacts, CompileAdapterContext, - CompiledLineageColumnFact, - CompiledModel, - CompileModelInput, - CompileProjectInputs, DynamicColumnContractProof, InferredColumn, ModelSqlAnalysis, ) +from sqlbuild.compiler.compile.types import CompiledModelView, CompileInputsView, ModelInput from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -424,13 +428,13 @@ def native_pivot_proofs(*, monkeypatch: pytest.MonkeyPatch) -> NativePivotProofs """Record every batch of native pivot proofs assembly takes.""" recorded_proofs: NativePivotProofs = NativePivotProofs() - native_proofs: Callable[..., tuple[DynamicColumnContractProof | None, ...]] = ( + native_proofs: Callable[..., tuple[ProofRow | None, ...]] = ( native_stage_assembly.prove_native_dynamic_contracts ) - def recorded(**keywords: Any) -> tuple[DynamicColumnContractProof | None, ...]: - proofs: tuple[DynamicColumnContractProof | None, ...] = native_proofs(**keywords) - recorded_proofs.proofs.extend(proofs) + def recorded(**keywords: Any) -> tuple[ProofRow | None, ...]: + proofs: tuple[ProofRow | None, ...] = native_proofs(**keywords) + recorded_proofs.proofs.extend(map(contract_proof, proofs)) proven: int = sum(proof is not None for proof in proofs) recorded_proofs.session_proofs += proven * (keywords["session"] is not None) return proofs @@ -514,7 +518,7 @@ def start(catalog: object, request: tuple[object, ...], *options: Any) -> Any: return started -def compile_inputs(*, project_dir: Path, files: dict[str, str]) -> CompileProjectInputs: +def compile_inputs(*, project_dir: Path, files: dict[str, str]) -> CompileInputsView: """Write a project and attach its compile inputs with the DuckDB compile context.""" for relative_path, contents in files.items(): @@ -574,14 +578,13 @@ class NativeAnalysisRuns: session_proofs: int = 0 proven_pivots: int = 0 native_enrichments: int = 0 - native_column_objects: int = 0 native_column_values: int = 0 golden: list[GoldenEntry] = field(default_factory=list) def analyse_natively( *, - inputs: CompileProjectInputs, + inputs: CompileInputsView, inference_profile: ExpressionInferenceProfile, lineage_mode: ColumnLineageMode, runs: NativeAnalysisRuns, @@ -595,13 +598,9 @@ def analyse_natively( `runs.golden` gathers the entries the Python oracle recorded for the same seam calls. """ - original_analysis: Callable[..., tuple[dict[str, ModelSqlAnalysis], Any]] = ( - native_stage_assembly.analyze_model_sql - ) + original_analysis: Callable[..., NativeModelAnalyses] = native_stage_assembly.analyze_model_sql - def analysed_three_ways( - request: NativeModelAnalysisRequest, - ) -> tuple[dict[str, ModelSqlAnalysis], Any]: + def analysed_three_ways(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: profile: ExpressionInferenceProfile = request.inference_profile cache: AnalysisCacheContext | None = build_analysis_cache_context( root=cache_root, inference_profile=profile, allow_compact_analysis=True @@ -616,13 +615,16 @@ def analysed_three_ways( inference_profile=replace(profile, binding_catalog=catalog), ) ) - cached_views.append((_analysis_views(cached.analyses), _catalog_view(catalog))) - analyses, session = original_analysis(replace(request, analysis_cache=None)) + cached_views.append( + (_analysis_views(_projected(cached, request, runs)), _catalog_view(catalog)) + ) + native: NativeModelAnalyses = original_analysis(replace(request, analysis_cache=None)) + session: Any = native.session + analyses: dict[str, ModelSqlAnalysis] = _projected(native, request, runs) uncached_view: object = (_analysis_views(analyses), _catalog_view(profile.binding_catalog)) columns: list[InferredColumn] = list( chain.from_iterable(map(_native_columns, analyses.values())) ) - runs.native_column_objects += len(set(map(id, columns))) runs.native_column_values += len(set(columns)) runs.analysed_models += len(analyses) runs.golden.append(golden_entry("models", _analysis_views(analyses))) @@ -630,7 +632,7 @@ def analysed_three_ways( for index, view in enumerate(cached_views): _append(runs, f"model analyses, cached run {index}", uncached_view, view) _compare_proofs(request=request, analyses=analyses, session=session, runs=runs) - return analyses, session + return native def recorded_shapes( *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile @@ -643,23 +645,14 @@ def recorded_shapes( runs.golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) return native - native_lineage_facts: Callable[..., tuple[CompiledLineageColumnFact, ...]] = ( - native_model_analysis.lineage_facts - ) - - def counted_native_enrichment(rows: list[Any]) -> tuple[CompiledLineageColumnFact, ...]: - runs.native_enrichments += 1 - return native_lineage_facts(rows) - - assembled_model: Callable[..., CompiledModel] = project_assembly._assemble_compiled_model + assembled_model: Callable[..., CompiledModelView] = project_assembly._assemble_compiled_model - def recorded_model(*arguments: Any, **keywords: Any) -> CompiledModel: - model: CompiledModel = assembled_model(*arguments, **keywords) + def recorded_model(*arguments: Any, **keywords: Any) -> CompiledModelView: + model: CompiledModelView = assembled_model(*arguments, **keywords) runs.golden.append(golden_entry("proof", model.dynamic_column_contract)) return model with monkeypatch.context() as patch: - patch.setattr(native_model_analysis, "lineage_facts", counted_native_enrichment) patch.setattr(project_assembly, "_assemble_compiled_model", recorded_model) patch.setattr(project_assembly, "analyze_model_sql", analysed_three_ways) patch.setattr(project_assembly, "expression_source_shapes_by_engine", recorded_shapes) @@ -693,7 +686,7 @@ def custom_nullability_rule(arguments: tuple[InferredNullability, ...]) -> Infer def analysis_request( *, - inputs: CompileProjectInputs, + inputs: CompileInputsView, monkeypatch: pytest.MonkeyPatch, inference_profile: ExpressionInferenceProfile = _DUCKDB_PROFILE, ) -> NativeModelAnalysisRequest: @@ -701,9 +694,9 @@ def analysis_request( requests: list[NativeModelAnalysisRequest] = [] - def captured(request: NativeModelAnalysisRequest) -> tuple[dict[str, ModelSqlAnalysis], None]: + def captured(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: requests.append(request) - return {}, None + return NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None) with ( monkeypatch.context() as patch, @@ -755,7 +748,7 @@ def _compare_proofs( and model_input.model_file.file_path.stem in analyses for model_input in request.model_inputs ] - pivot_inputs: list[CompileModelInput] = list(itertools.compress(request.model_inputs, proved)) + pivot_inputs: list[ModelInput] = list(itertools.compress(request.model_inputs, proved)) for model_input in pivot_inputs: families: tuple[SchemaDynamicColumnFamily, ...] = model_dynamic_families(model_input) analysis: ModelSqlAnalysis = analyses[model_input.model_file.file_path.stem] @@ -772,12 +765,12 @@ def _compare_proofs( ), ) proof: DynamicColumnContractProof | None = analysis.dynamic_column_contract - standalone: DynamicColumnContractProof | None = prove_native_dynamic_contracts( - session=None, tables=tables, models=models - )[0] - in_session: DynamicColumnContractProof | None = prove_native_dynamic_contracts( - session=session, tables=tables, models=models - )[0] + standalone: DynamicColumnContractProof | None = contract_proof( + prove_native_dynamic_contracts(session=None, tables=tables, models=models)[0] + ) + in_session: DynamicColumnContractProof | None = contract_proof( + prove_native_dynamic_contracts(session=session, tables=tables, models=models)[0] + ) runs.pivot_proofs += proof is not None runs.proven_pivots += proof is not None and proof.output_proven runs.standalone_proofs += standalone is not None @@ -792,6 +785,28 @@ def _append(runs: NativeAnalysisRuns, name: str, uncached: object, cached: objec runs.cached.append(cached) +def _projected( + native: NativeModelAnalyses, request: NativeModelAnalysisRequest, runs: NativeAnalysisRuns +) -> dict[str, ModelSqlAnalysis]: + """The test oracle's projection of each finished outcome as the analysis it replaced.""" + + placeholders: dict[str, dict[str, str] | None] = { + model_input.model_file.file_path.stem: model_placeholders(model_input) + for model_input in request.model_inputs + } + sources: list[str] = [ + cast(Any, native.session).outcome_rows(position)[0][2] + for position in native.positions.values() + ] + runs.native_enrichments += sum(source != LINEAGE_NATIVE for source in sources) + return { + name: session_outcome_analysis( + session=native.session, position=position, placeholders=placeholders.get(name) + ) + for name, position in native.positions.items() + } + + def _analysis_views(analyses: dict[str, ModelSqlAnalysis]) -> object: return { name: ( diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 6a1a488a99..93a1a26140 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -20,10 +20,7 @@ ) from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile.models import ( - CompiledProject, - CompileProjectInputs, -) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.types import CompilerEngine from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability @@ -197,7 +194,7 @@ def test_given_generated_projects_when_analysing_natively_then_cached_and_proven rng: random.Random = random.Random(test_case.seed) runs: NativeAnalysisRuns = NativeAnalysisRuns() for index in range(test_case.count): - inputs: CompileProjectInputs = compile_inputs( + inputs: CompileInputsView = compile_inputs( project_dir=tmp_path / f"project_{index}", files={ **generated_analysis_files(rng=rng, model_count=test_case.model_count), @@ -231,7 +228,7 @@ def test_given_generated_projects_when_analysing_natively_then_cached_and_proven assert runs.session_proofs >= test_case.expected_minimum_pivot_proofs assert runs.proven_pivots >= test_case.expected_minimum_proven_pivots assert runs.native_enrichments >= test_case.expected_minimum_native_enrichments - assert runs.native_column_objects == runs.native_column_values > 0 + assert runs.native_column_values > 0 @pytest.mark.parametrize( @@ -256,7 +253,7 @@ def test_given_equal_model_queries_when_analysing_natively_then_shares_and_cache ) -> None: record_dir: Path = tmp_path / "records" monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) - inputs: CompileProjectInputs = compile_inputs( + inputs: CompileInputsView = compile_inputs( project_dir=tmp_path / "project", files=shared_analysis_files( regions=test_case.regions, inexact_regions=test_case.inexact_regions @@ -293,7 +290,6 @@ def test_given_equal_model_queries_when_analysing_natively_then_shares_and_cache sum(session.sharing[1] for session in uncached), runs.analysed_models - shared, deferral_kinds(record_dir), - runs.native_column_objects, runs.native_column_values, ) == ( test_case.expected_analysed, @@ -302,7 +298,6 @@ def test_given_equal_model_queries_when_analysing_natively_then_shares_and_cache test_case.expected_unshared, Counter(), test_case.expected_column_values, - test_case.expected_column_values, ) @@ -324,7 +319,7 @@ def test_given_cyclic_models_when_analysing_natively_then_session_answers_and_ca ) -> None: record_dir: Path = tmp_path / "records" monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) - inputs: CompileProjectInputs = compile_inputs( + inputs: CompileInputsView = compile_inputs( project_dir=tmp_path / "project", files=test_case.files ) sessions: list[Any] = started_sessions(monkeypatch=monkeypatch) @@ -423,7 +418,7 @@ def test_given_internal_native_failure_when_assembling_then_raises_without_pytho ) -> None: record_dir: Path = tmp_path / "records" monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) - inputs: CompileProjectInputs = compile_inputs( + inputs: CompileInputsView = compile_inputs( project_dir=tmp_path / "project", files=test_case.files ) duplicate_analysed_model_names(monkeypatch=monkeypatch) @@ -471,13 +466,13 @@ def test_given_internal_native_failure_when_assembling_then_raises_without_pytho def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_without_the_wheel( test_case: StandalonePivotProofTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - inputs: CompileProjectInputs = compile_inputs( + inputs: CompileInputsView = compile_inputs( project_dir=tmp_path / "project", files=pivot_project_files() ) monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.NATIVE_PREVIEW.value) recorded: NativePivotProofs = native_pivot_proofs(monkeypatch=monkeypatch) - project: CompiledProject = assemble_compiled_project( + project: CompiledProjectView = assemble_compiled_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb"), analysis_model_names=test_case.analysed_models, @@ -522,7 +517,7 @@ def test_given_adapter_nullability_rule_when_analysing_natively_then_rule_is_cal record_dir: Path = tmp_path / "records" monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) - assert analyze_native_model_sql(request=request).analyses + assert analyze_native_model_sql(request=request).positions assert len(rule.calls) >= test_case.expected_minimum_calls, test_case.description assert all( isinstance(value, InferredNullability) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index 435209c048..cc21e4bde6 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -17,9 +17,9 @@ from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( infer_native_query_columns, ) -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery +from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeModelAnalyses from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( FallbackColumnsTestCase, MissingSessionAnalysisTestCase, @@ -56,14 +56,14 @@ def test_given_unprojectable_model_when_compiling_then_parsed_tree_columns_are_kept( test_case: FallbackColumnsTestCase, tmp_path: Path ) -> None: - project: CompiledProject = compiled_project( + project: CompiledProjectView = compiled_project( project_dir=tmp_path, files={ "sqlbuild_project.toml": _PROJECT, "models/orders.sql": _HEADER + test_case.query_sql, }, ) - model: CompiledModel = project.models[0] + model: CompiledModelView = project.models[0] assert model.inferred_columns is not None assert ( @@ -131,7 +131,11 @@ def test_given_table_function_when_compiling_then_body_columns_match_declaration def test_given_session_without_model_analysis_when_assembling_then_raises_native_compiler_error( test_case: MissingSessionAnalysisTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setattr(project_assembly, "analyze_model_sql", lambda _request: ({}, None)) + monkeypatch.setattr( + project_assembly, + "analyze_model_sql", + lambda _request: NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None), + ) with pytest.raises(native_module.NativeCompilerError) as raised: _ = compiled_project( diff --git a/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py index 3ee6f94f54..72889e7c71 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py @@ -3,18 +3,6 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class AttachedAuditParityTestCase: - """Seeded attached audits rendered natively and by Python's attachment helpers.""" - - description: str - seed: int - count: int - expected_minimum_native: int - expected_minimum_native_errors: int - expected_minimum_python_errors: int - - @dataclass(frozen=True) class BodyCallParityTestCase: """Seeded helper bodies whose call reading the native extractor shares with a Python scanner.""" diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index 5f00772f85..57a93eed0a 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -2,25 +2,14 @@ from __future__ import annotations +import json import random import re -from dataclasses import dataclass -from pathlib import Path -from typing import cast +from typing import Any -from sqlbuild.compiler.attachments.main._render_native_attached_audit import ( - render_native_attached_audit, -) -from sqlbuild.compiler.attachments.models import NativeAuditPolicies, NativeRenderedAudit -from sqlbuild.compiler.auditing.types import AuditRunScope, AuditSeverity -from sqlbuild.compiler.compile._helpers.attachment.audits import ( - resolve_audit_run_scope, - resolve_audit_severity, -) +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.refs.references import scan_sql_reference_calls -from sqlbuild.compiler.compile._helpers.render.arguments import render_parameterized_sql from sqlbuild.compiler.compile._helpers.render.macros import find_macro_call_names -from sqlbuild.compiler.compile._helpers.sql_tests.native import extract_unexpanded_sql_test from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( SqlReferenceScan, @@ -47,167 +36,6 @@ "@'values'\u3000", "\n", ) -_VALUES: tuple[object, ...] = ( - "status", - "it's", - 1, - 1.0, - -0.0, - True, - None, - ["placed", 2, None], - ("placed",), - ("placed", 3.5, float("inf")), - [("placed",), ["shipped"]], - {"nested": 1}, - float("nan"), - AuditSeverity.WARN, - 10**20, -) -_SEVERITIES: tuple[str | None, ...] = (None, None, None, "warn", "error", AuditSeverity.WARN) -_RUN_SCOPES: tuple[str | None, ...] = (None, None, None, "final", "delta_and_final") -_RARE: tuple[tuple[str, object], ...] = ( - ("sql", "@missing"), - ("severity", "fatal"), - ("severity", ""), - ("run_scope", "delta"), - ("column", "amount"), - ("column", 1), - ("column", ("status",)), - ("column", ["amount"]), -) - - -@dataclass(frozen=True) -class GeneratedAudit: - """One attachment: SQL, arguments and authored policies.""" - - sql_body: str - evidence_sql: str - implicit_arguments: dict[str, str] - explicit_arguments: dict[object, object] - instance_severity: str | None - default_severity: str | None - instance_run_scope: str | None - default_run_scope: str | None - - -@dataclass(frozen=True) -class AuditParity: - """Python's rendering (or error text) and the native rendering.""" - - audit: GeneratedAudit - python: tuple[object, ...] | str - native: NativeRenderedAudit - - -def generated_audit(*, rng: random.Random) -> GeneratedAudit: - """Return one attachment mixing parameter shapes, argument values and policies.""" - - rare: dict[str, object] = dict(rng.choices(_RARE, k=int(rng.random() < 0.3))) - explicit: dict[object, object] = { - "values": rng.choice(_VALUES), - "limit_rows": rng.choice(_VALUES), - } - explicit.update(filter(_is_column_override, rare.items())) - explicit.update(dict.fromkeys(rng.choices((7,), k=int(rng.random() < 0.1)), "x")) - return GeneratedAudit( - sql_body="".join(rng.choices(_SQL_PIECES, k=rng.randint(1, 8))) + str(rare.get("sql", "")), - evidence_sql="SELECT @column FROM t", - implicit_arguments={"column": rng.choice(("status", "amount"))}, - explicit_arguments=explicit, - instance_severity=cast(str | None, rare.get("severity", rng.choice(_SEVERITIES))), - default_severity=rng.choice(_SEVERITIES), - instance_run_scope=cast(str | None, rare.get("run_scope", rng.choice(_RUN_SCOPES))), - default_run_scope=rng.choice(_RUN_SCOPES), - ) - - -def _is_column_override(item: tuple[str, object]) -> bool: - return item[0] == "column" - - -def audit_parity(audit: GeneratedAudit) -> AuditParity: - """Render one attachment with Python's helpers and natively.""" - - return AuditParity( - audit=audit, - python=_python_rendering(audit), - native=render_native_attached_audit( - labels=("models/orders.sql", "floor"), - sql_body=audit.sql_body, - evidence_sql=audit.evidence_sql, - implicit_arguments=audit.implicit_arguments, - explicit_arguments=cast(dict[str, object], audit.explicit_arguments), - policies=NativeAuditPolicies( - has_thresholds=False, - threshold_error=False, - instance_severity=audit.instance_severity, - default_severity=audit.default_severity, - instance_run_scope=audit.instance_run_scope, - default_run_scope=audit.default_run_scope, - ), - ), - ) - - -def native_outcome(parity: AuditParity) -> tuple[object, ...] | str: - """The native rendering or error spelled like Python's, with the run scope it selects.""" - - rendered: NativeRenderedAudit = parity.native - return rendered.render_error or rendered.policy_error or _native_rendering(parity) - - -def _native_rendering(parity: AuditParity) -> tuple[object, ...]: - rendered: NativeRenderedAudit = parity.native - run_scope: str | None = { - "instance": parity.audit.instance_run_scope, - "default": parity.audit.default_run_scope, - }.get(rendered.run_scope_source, AuditRunScope.DELTA_AND_FINAL) - return (rendered.sql_body, rendered.evidence_sql, AuditSeverity(rendered.severity), run_scope) - - -def _python_rendering(audit: GeneratedAudit) -> tuple[object, ...] | str: - owner: Path = Path("models/orders.sql") - implicit: dict[object, object] = cast(dict[object, object], audit.implicit_arguments) - overrides: list[object] = [ - name - for name, value in filter( - lambda item: item[0] in implicit and implicit[item[0]] != item[1], - audit.explicit_arguments.items(), - ) - ] - try: - for name in overrides[:1]: - raise CompileInputError( - f"{owner} audit 'floor' must not override implicit {name} from attached context" - ) - merged: dict[str, object] = cast( - dict[str, object], {**audit.implicit_arguments, **audit.explicit_arguments} - ) - rendered: tuple[str, ...] = tuple( - render_parameterized_sql( - sql=sql, - arguments=merged, - owner_label=str(owner), - definition_label="generic audit 'floor'", - ) - for sql in (audit.sql_body, audit.evidence_sql) - ) - severity: AuditSeverity = resolve_audit_severity( - instance_severity=audit.instance_severity, - default_severity=audit.default_severity, - audit_label=f"{owner} audit 'floor'", - ) - run_scope: str = resolve_audit_run_scope( - instance_run_scope=audit.instance_run_scope, - default_run_scope=audit.default_run_scope, - ) - except CompileInputError as error: - return str(error) - return (*rendered, severity, run_scope) - - _MACRO_PIECES: tuple[str, ...] = ( "a", "@m(1)", @@ -315,19 +143,16 @@ def macro_scannable(body: str) -> bool: def native_macro_outcome(*, body: str) -> str: """Whether the raw extractor rejects a macro-test helper `body` for calling macros.""" - try: - _ = extract_unexpanded_sql_test( - sql=( - f"WITH h AS ({body}), __macro_actual__ AS (SELECT @m(1) AS a), " - "__macro_expected__ AS (SELECT 1 AS a)" - ), - file_label="tests/unit/test_logic.sql", - mode=SqlTestMode.MACRO, - syntax=SqlLexicalSyntax(), - ) - except CompileInputError as error: - return f"calls macros {'must not call macros' in str(error)}" - return "calls macros False" + response: dict[str, Any] = _raw_extraction( + sql=( + f"WITH h AS ({body}), __macro_actual__ AS (SELECT @m(1) AS a), " + "__macro_expected__ AS (SELECT 1 AS a)" + ), + mode=SqlTestMode.MACRO, + syntax=SqlLexicalSyntax(), + ) + message: str = str(response.get("error", {}).get("message", "")) + return f"calls macros {'must not call macros' in message}" def python_reference_outcome(*, body: str, syntax: SqlLexicalSyntax) -> str: @@ -345,17 +170,41 @@ def python_reference_outcome(*, body: str, syntax: SqlLexicalSyntax) -> str: def native_reference_outcome(*, body: str, syntax: SqlLexicalSyntax) -> str: """The raw extractor's reading of a UDF-test helper `body`, as `python_reference_outcome`.""" - try: - _, invalid_calls = extract_unexpanded_sql_test( - sql=( - f"WITH h AS ({body}), __udf_actual__ AS (SELECT 1 AS a), " - "__udf_expected__ AS (SELECT 1 AS a)" - ), - file_label="tests/unit/test_logic.sql", - mode=SqlTestMode.UDF, - syntax=syntax, - ) - except CompileInputError as error: - called: re.Match[str] | None = re.search(r"must not call (\w+);", str(error)) - return (called and f"calls {called.group(1)}") or f"error: {error}" - return f"malformed calls {invalid_calls}" + response: dict[str, Any] = _raw_extraction( + sql=( + f"WITH h AS ({body}), __udf_actual__ AS (SELECT 1 AS a), " + "__udf_expected__ AS (SELECT 1 AS a)" + ), + mode=SqlTestMode.UDF, + syntax=syntax, + ) + message: str = str(response.get("error", {}).get("message", "")) + called: re.Match[str] | None = re.search(r"must not call (\w+);", message) + invalid_calls: object = response.get("tests", [{}])[0].get("invalidCalls", False) + return ( + (called and f"calls {called.group(1)}") + or (message and f"error: {message}") + or f"malformed calls {invalid_calls}" + ) + + +def _raw_extraction(*, sql: str, mode: SqlTestMode, syntax: SqlLexicalSyntax) -> dict[str, Any]: + """The native extractor's response for one test before expansion.""" + + request: dict[str, object] = { + "syntax": {_SYNTAX_FIELDS[key]: value for key, value in syntax.native_mapping.items()}, + "tests": [ + {"sql": sql, "fileLabel": "tests/unit/test_logic.sql", "mode": mode.value, "raw": True} + ], + } + return json.loads(_native.extract_sql_tests_json(json.dumps(request))) + + +_SYNTAX_FIELDS: dict[str, str] = { + "backslash_escape_quotes": "backslashEscapeQuotes", + "escape_string_prefix": "escapeStringPrefix", + "raw_string_prefix": "rawStringPrefix", + "triple_quoted_strings": "tripleQuotedStrings", + "nested_block_comments": "nestedBlockComments", + "line_comment_prefixes": "lineCommentPrefixes", +} diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_audit_rendering.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_audit_rendering.py deleted file mode 100644 index be46a71f85..0000000000 --- a/tests/integration/src/sqlbuild/compiler/attachments/test_native_audit_rendering.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Natively rendered attached audits match Python's argument merge, SQL, policies and errors.""" - -from __future__ import annotations - -import random - -import pytest - -from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( - AttachedAuditParityTestCase, -) -from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( - AuditParity, - audit_parity, - generated_audit, - native_outcome, -) -from tests.integration.src.sqlbuild.compiler.helpers import mismatches - - -@pytest.mark.parametrize( - "test_case", - [ - AttachedAuditParityTestCase( - description="seeded parameters, argument values, overrides and policies", - seed=20261008, - count=5000, - expected_minimum_native=5000, - expected_minimum_native_errors=600, - expected_minimum_python_errors=1000, - ), - ], - ids=lambda case: case.description, -) -def test_given_generated_attachments_when_rendering_natively_then_python_rendering_matches( - test_case: AttachedAuditParityTestCase, -) -> None: - rng: random.Random = random.Random(test_case.seed) - - parities: list[AuditParity] = [ - audit_parity(generated_audit(rng=rng)) for _ in range(test_case.count) - ] - - assert ( - mismatches( - inputs=[parity.audit for parity in parities], - expected=[parity.python for parity in parities], - actual=list(map(native_outcome, parities)), - ), - len(parities) >= test_case.expected_minimum_native, - sum(isinstance(native_outcome(parity), str) for parity in parities) - >= test_case.expected_minimum_native_errors, - sum(isinstance(parity.python, str) for parity in parities) - >= test_case.expected_minimum_python_errors, - ) == ([], True, True, True), test_case.description - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/compile/_test_types.py b/tests/integration/src/sqlbuild/compiler/compile/_test_types.py index f75f91d1ab..bddab3ba58 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_test_types.py @@ -64,12 +64,3 @@ class FailingMacroProjectParityTestCase: expected_minimum_errors_after_macro_runs: int expected_minimum_distinct_errors: int expected_mismatches: list[tuple[object, object, object]] - - -@dataclass(frozen=True) -class DeclarationUsageTestCase: - """SQL texts whose declaration usages Python's usage walk recorded in a golden.""" - - description: str - texts: tuple[str, ...] - expected_differences: list[tuple[int, str, object, object]] diff --git a/tests/integration/src/sqlbuild/compiler/compile/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/helpers.py index 1a104bc0fa..8de80c74e7 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/helpers.py @@ -15,18 +15,16 @@ import pytest from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.render.declarations import declaration_usage_records from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros_result from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, - CompileProjectInputs, - DeclarationResolutionContext, LoadedMacro, MacroContext, MacroExpansionResult, ) +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture @@ -39,8 +37,8 @@ from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.constants import ACTIVE_MACRO_BRIDGE -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity, VisibilityRecord -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, VisibilityReason +from sqlbuild.compiler.scopes.models import ResourceIdentity +from sqlbuild.compiler.scopes.types import ResourceKind from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( resolve_effective_collection_rendering, ) @@ -232,7 +230,7 @@ def write_project(*, root: Path, files: dict[str, str]) -> Path: def render_compile_inputs( *, project_dir: Path, engine: str, monkeypatch: pytest.MonkeyPatch -) -> CompileProjectInputs: +) -> CompileInputsView: """Render the project's compile inputs through the frontier under one compiler engine.""" with monkeypatch.context() as patch: @@ -261,7 +259,7 @@ def render_compile_inputs( ) -def comparable_capture(inputs: CompileProjectInputs) -> str: +def comparable_capture(inputs: CompileInputsView) -> str: """Render compile inputs as canonical JSON without the per-run identifier.""" return STAGE_CAPTURE_INVOCATION_ID_PATTERN.sub( @@ -558,43 +556,3 @@ def engine_render_outcome( ), }[failure is None] return outcome, tuple(log_path.read_text(encoding="utf-8").splitlines()) - - -_USAGE_CONSUMER: ResourceIdentity = ResourceIdentity(ResourceKind.TEST, "orders_test") - - -def _visible(kind: DeclarationKind, name: str) -> tuple[VisibilityRecord, ...]: - return ( - VisibilityRecord( - resource=_USAGE_CONSUMER, - declaration=DeclarationIdentity(kind, name), - reason=VisibilityReason.GLOBAL, - ), - ) - - -_USAGE_DECLARATIONS: DeclarationResolutionContext = DeclarationResolutionContext( - enum_visibility={ - "status": _visible(DeclarationKind.ENUM, "status"), - "size": _visible(DeclarationKind.ENUM, "size"), - }, - constant_visibility={"rate": _visible(DeclarationKind.CONSTANT, "rate")}, -) - - -def declaration_usage_outcome(sql: str) -> tuple[object, ...]: - """The names of the declarations `sql` uses, in order, or the error the scan raises.""" - - capture: ExpansionFailureCapture = ExpansionFailureCapture() - names: list[tuple[object, ...]] = [()] - with capture: - names[0] = tuple( - usage.declaration.name - for usage in declaration_usage_records( - sql=sql, resource=_USAGE_CONSUMER, declarations=_USAGE_DECLARATIONS - ) - ) - return { - True: names[0], - False: (type(capture.failure).__name__, str(capture.failure)), - }[capture.failure is None] diff --git a/tests/integration/src/sqlbuild/compiler/compile/test_native_declaration_usages.py b/tests/integration/src/sqlbuild/compiler/compile/test_native_declaration_usages.py deleted file mode 100644 index 7f84016974..0000000000 --- a/tests/integration/src/sqlbuild/compiler/compile/test_native_declaration_usages.py +++ /dev/null @@ -1,68 +0,0 @@ -"""SQL-test declaration usages come from the native reference scan and match Python's walk.""" - -from __future__ import annotations - -import pytest - -from tests.integration.src.sqlbuild.compiler.compile._test_types import DeclarationUsageTestCase -from tests.integration.src.sqlbuild.compiler.compile.helpers import declaration_usage_outcome -from tests.integration.src.sqlbuild.compiler.helpers import ( - GoldenEntry, - golden_differences, - golden_entry, - golden_name, - read_golden, -) - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationUsageTestCase( - description="usage scan edges", - texts=( - "SELECT @enum('status').OPEN, @const('rate')", - "SELECT @enum('status') , @const('rate')", - "SELECT @enum x 'unclosed", - "SELECT @enum x, @const('rate') 'unclosed", - 'SELECT @const(rate), @enum("size").S', - "SELECT '@enum(''status'').A', @const('rate') /* open", - "SELECT @enumx('status').A, @const_x('rate'), @consté, @const('rate')", - "SELECT $$ @enum('status').A $$, @enum('size').M -- @const('rate')\n", - "SELECT x$$ @const('rate') $$", - "SELECT @enum ( 'status' ) . OPEN, @enum('status').OPEN, @const('missing')", - "SELECT `@const('rate')`, \"@enum('size').A\"", - "SELECT @const('rate'), @const('rate'), 'é' @enum('size').XL", - "SELECT @@const, @enum, @const", - "SELECT 1", - "SELECT @enum('status').OPEN /* @const('rate') */ 'it''s' @const('rate')", - "SELECT @const('rate' ", - "SELECT /* x @const('rate')", - "SELECT 'a @enum('status').A", - "SELECT @enum x 'oops @const('rate')", - "SELECT @const('rate') $tag$ @enum('size').S", - "SELECT @enum(\"status\").OPEN, @const('rate') /* tail @enum('size').S", - ), - expected_differences=[], - ), - ], - ids=lambda case: case.description, -) -def test_given_declaration_references_when_collecting_usages_then_matches_python_golden( - test_case: DeclarationUsageTestCase, -) -> None: - outcomes: list[tuple[object, ...]] = [declaration_usage_outcome(sql) for sql in test_case.texts] - expected: list[GoldenEntry] = read_golden( - golden_name("declaration_usages", test_case.description) - ) - - assert ( - golden_differences( - expected, [golden_entry("declaration_usages", outcome) for outcome in outcomes] - ) - == test_case.expected_differences - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/__init__.py b/tests/integration/src/sqlbuild/compiler/compiled_project/__init__.py new file mode 100644 index 0000000000..9cfb90ca22 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/__init__.py @@ -0,0 +1 @@ +"""Native compiled project integration tests.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py new file mode 100644 index 0000000000..b076b00c34 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -0,0 +1,181 @@ +"""Test case types for native compiled project integration tests.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class RetainedRulesRowsTestCase: + """A real compile whose rules request reads model facts the compile-input stage retained.""" + + description: str + expected_codes: tuple[str, ...] + + +@dataclass(frozen=True) +class UnretainedRulesRowsTestCase: + """A compile whose model facts were not retained, so rules cannot read them.""" + + description: str + expected_error: str + + +@dataclass(frozen=True) +class ZeroMaterializationTestCase: + """An ordinary compile that must not construct the legacy Python project representation.""" + + description: str + arguments: tuple[str, ...] + expected_constructions: dict[str, int] + + +@dataclass(frozen=True) +class ModelLoopParityTestCase: + """A project whose model inputs the native loop builds as the attachment recorded them.""" + + description: str + fixtures: tuple[str, ...] + files: dict[str, str] + expected_native_models: int + expected_summary: str + + +@dataclass(frozen=True) +class ModelLoopErrorTestCase: + """A model the native loop rejects with the attachment's exact message and code.""" + + description: str + model_sql: str + expected_message: str + expected_code: str + files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ModelPathScopeErrorTestCase: + """A model path the scope lookup rejects while the loop resolves its declarations.""" + + description: str + model_path: str + expected_message: str + expected_code: str + + +@dataclass(frozen=True) +class ImmediateSyntaxErrorTestCase: + """A model the native loop rejects while validating its SQL syntax immediately.""" + + description: str + model_sql: str + expected_message: str + expected_help_fragment: str + + +@dataclass(frozen=True) +class NativeSourceAttachmentTestCase: + """A source the native loop attaches: its assembled entry and recorded declaration usages.""" + + description: str + source_yml: str + source_name: str + expected_entry: dict[str, object] + expected_usages: tuple[str, ...] + + +@dataclass(frozen=True) +class NativeSourceErrorTestCase: + """A source the native loop rejects while attaching it.""" + + description: str + source_yml: str + expected_message: str + expects_help: bool + + +@dataclass(frozen=True) +class NativeFunctionAttachmentTestCase: + """A function the native loop attaches and assembly keeps in the native project.""" + + description: str + function_name: str + expected_summary: tuple[object, ...] + expected_usages: tuple[str, ...] + + +@dataclass(frozen=True) +class NativeFunctionErrorTestCase: + """Function files the native loop rejects while attaching them.""" + + description: str + files: dict[str, str] + expected_message: str + + +@dataclass(frozen=True) +class NativeAuditAttachmentTestCase: + """An audit the native loop attaches and assembly keeps in the native project.""" + + description: str + audit_name: str + expected_summary: tuple[object, ...] + expected_usages: tuple[str, ...] + + +@dataclass(frozen=True) +class NativeAuditErrorTestCase: + """Audit files the native loop rejects while attaching them.""" + + description: str + files: dict[str, str] + expected_message: str + + +@dataclass(frozen=True) +class NativeMeasurementPolicyTestCase: + """A standalone measurement audit whose header policy the native loop parses.""" + + description: str + files: dict[str, str] + expected_policy: tuple[object, ...] + + +@dataclass(frozen=True) +class NativeResourceDiagnosticTestCase: + """A project's files and the P010, P011 and C216 diagnostics compiling it reports.""" + + description: str + files: dict[str, str] + expected_diagnostics: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class HandBuiltAnalysisTestCase: + """Model SQL and the analysis facts native and hand-built assembly report for it.""" + + description: str + models: dict[str, str] + expected_models: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SharedInferredColumnsTestCase: + description: str + model_count: int + expected_columns_per_model: int + + +@dataclass(frozen=True) +class AdapterParameterBoundaryTestCase: + description: str + adapter_name: str + declaration: str + literal: str + expected_sql: str + + +@dataclass(frozen=True) +class QuotedHookBoundaryTestCase: + description: str + argument: str + expected_sql: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/header_contract.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/header_contract.json new file mode 100644 index 0000000000..d0c8c20319 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/header_contract.json @@ -0,0 +1,60 @@ +{ + "diagnostics": [ + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'category_amounts' has no description\", resource_type=, resource_name='category_amounts', column_name=None, path=PosixPath('models/category_amounts.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/category_amounts.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'category_amounts\\', add this to the MODEL header in models/category_amounts.sql:\\n MODEL (\\n description \"What one row of category_amounts represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'order_amounts' has no description\", resource_type=, resource_name='order_amounts', column_name=None, path=PosixPath('models/order_amounts.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/order_amounts.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'order_amounts\\', add this to the MODEL header in models/order_amounts.sql:\\n MODEL (\\n description \"What one row of order_amounts represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'order_statuses' has no description\", resource_type=, resource_name='order_statuses', column_name=None, path=PosixPath('models/order_statuses.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/order_statuses.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'order_statuses\\', add this to the MODEL header in models/order_statuses.sql:\\n MODEL (\\n description \"What one row of order_statuses represents\",\\n ...\\n );', notes=(), affected_rules=())" + ], + "models": { + "models/category_amounts.sql": { + "config": "CompileModelConfig(values={}, model_header_keys=('columns', 'dynamic_columns'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT * FROM __ref(\"order_amounts\")\\nPIVOT(MAX(amount) FOR category IN (ANY ORDER BY category))', expanded_sql='SELECT * FROM __ref(\"order_amounts\")\\nPIVOT(MAX(amount) FOR category IN (ANY ORDER BY category))', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT * FROM __ref(\"order_amounts\")\\nPIVOT(MAX(amount) FOR category IN (ANY ORDER BY category))'", + "macro_usages": "()", + "query_sql": "'SELECT * FROM __ref(\"order_amounts\")\\nPIVOT(MAX(amount) FOR category IN (ANY ORDER BY category))'", + "references": "(CompileSqlReference(ref_kind=, ref_name='order_amounts', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='category_amounts', model_schema=None, description=None, type_enforcement=True, meta={}, columns=(SchemaColumn(name='customer_id', type='INTEGER', nullable=None, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/category_amounts.sql'), line=2, column=12, end_line=2, end_column=23), migrate_from=None),), dynamic_columns=(SchemaDynamicColumnFamily(name='category_amounts', pivot_column='category', value_column='amount', aggregate='MAX', type='INTEGER', name_pattern='^[a-z]+$'),), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT * FROM __ref(\"order_amounts\")\\nPIVOT(MAX(amount) FOR category IN (ANY ORDER BY category))', expanded_sql='SELECT * FROM __ref(\"order_amounts\")\\nPIVOT(MAX(amount) FOR category IN (ANY ORDER BY category))', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/order_amounts.sql": { + "config": "CompileModelConfig(values={}, model_header_keys=('columns',), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql=\"SELECT 1 AS customer_id, 'books' AS category, 2 AS amount\", expanded_sql=\"SELECT 1 AS customer_id, 'books' AS category, 2 AS amount\", passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "\"SELECT 1 AS customer_id, 'books' AS category, 2 AS amount\"", + "macro_usages": "()", + "query_sql": "\"SELECT 1 AS customer_id, 'books' AS category, 2 AS amount\"", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='order_amounts', model_schema=None, description=None, type_enforcement=True, meta={}, columns=(SchemaColumn(name='customer_id', type='INTEGER', nullable=None, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/order_amounts.sql'), line=1, column=17, end_line=1, end_column=28), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql=\"SELECT 1 AS customer_id, 'books' AS category, 2 AS amount\", expanded_sql=\"SELECT 1 AS customer_id, 'books' AS category, 2 AS amount\", passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/order_statuses.sql": { + "config": "CompileModelConfig(values={'contract': 'enforced'}, model_header_keys=('audit_factories', 'columns', 'contract'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='order_statuses'), declaration=DeclarationIdentity(kind=, name='order_status', owner=None), kind=, through=None, enum_member='PLACED'), UsageRecord(consumer=ResourceIdentity(kind=, name='order_statuses'), declaration=DeclarationIdentity(kind=, name='order_status', owner=None), kind=, through=None, enum_member=None))", + "enum_columns": "{'status': EnumDeclaration(name='order_status', members=(EnumMember(name='PLACED', value='PLACED'), EnumMember(name='SHIPPED', value='SHIPPED')), scalar_type='VARCHAR', relative_path=PosixPath('enums/status.sql'), model_name=None)}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT @enum(\"order_status\").PLACED AS status, 3 AS total', expanded_sql=\"SELECT 'PLACED' AS status, 3 AS total\", passes=((), (ExpansionSpan(source_start=7, source_end=35, output_start=7, output_end=15),), ()))", + "macro_deps": "()", + "macro_source_sql": "\"SELECT 'PLACED' AS status, 3 AS total\"", + "macro_usages": "()", + "query_sql": "\"SELECT 'PLACED' AS status, 3 AS total\"", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='order_statuses', model_schema=None, description=None, type_enforcement=True, meta={}, columns=(SchemaColumn(name='status', type='VARCHAR', nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None), SchemaAuditInstance(definition_name='accepted_values', arguments={'values': ('PLACED', 'SHIPPED')}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None)), location=SourceLocation(path=PosixPath('models/order_statuses.sql'), line=3, column=12, end_line=3, end_column=18), migrate_from=None), SchemaColumn(name='total', type='INTEGER', nullable=None, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/order_statuses.sql'), line=3, column=59, end_line=3, end_column=64), migrate_from=None)), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'total > 0'}, name='positive_total', description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT @enum(\"order_status\").PLACED AS status, 3 AS total', expanded_sql=\"SELECT 'PLACED' AS status, 3 AS total\", passes=((ExpansionSpan(source_start=7, source_end=35, output_start=7, output_end=15),), ()))", + "sql_validation_enabled": "True" + } + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/model_schema.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/model_schema.json new file mode 100644 index 0000000000..ea77bd3b77 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/model_schema.json @@ -0,0 +1,41 @@ +{ + "diagnostics": [ + "CompilerDiagnostic(phase=, severity=, code='S006', message=\"Schema 'marts_shape' is not visible from 'models/staging/hidden.sql'. It is defined at models/marts/_sqlbuild/schemas/local.sql with inherited scope owned by 'models/marts'\", resource_type=, resource_name='hidden', column_name=None, path=PosixPath('models/staging/hidden.sql'), line=None, column=None, location=None, related_locations=(), help=\"move 'models/marts/_sqlbuild/schemas/local.sql' to a schemas/ role owned by 'models/staging' or one of its parent folders, or to the project-wide schemas/ when consumers span resource trees\", notes=(), affected_rules=())" + ], + "models": { + "models/orders.sql": { + "config": "CompileModelConfig(values={'contract': 'enforced'}, model_header_keys=('columns', 'contract', 'model_schema'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='orders'), declaration=DeclarationIdentity(kind=, name='order_status', owner=None), kind=, through=None, enum_member=None), UsageRecord(consumer=ResourceIdentity(kind=, name='orders'), declaration=DeclarationIdentity(kind=, name='order_shape', owner=None), kind=, through=None, enum_member=None))", + "enum_columns": "{'status': EnumDeclaration(name='order_status', members=(EnumMember(name='PLACED', value='PLACED'), EnumMember(name='SHIPPED', value='SHIPPED')), scalar_type='VARCHAR', relative_path=PosixPath('enums/status.sql'), model_name=None)}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql=\"SELECT 1 AS order_id, 'PLACED' AS status, 'x' AS note\", expanded_sql=\"SELECT 1 AS order_id, 'PLACED' AS status, 'x' AS note\", passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "\"SELECT 1 AS order_id, 'PLACED' AS status, 'x' AS note\"", + "macro_usages": "()", + "query_sql": "\"SELECT 1 AS order_id, 'PLACED' AS status, 'x' AS note\"", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='orders', model_schema='order_shape', description='Order shape.', type_enforcement=True, meta={}, columns=(SchemaColumn(name='order_id', type='INTEGER', nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('schemas/orders.sql'), line=4, column=12, end_line=4, end_column=20), migrate_from=None), SchemaColumn(name='status', type='VARCHAR', nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='accepted_values', arguments={'values': ('PLACED', 'SHIPPED')}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('schemas/orders.sql'), line=4, column=53, end_line=4, end_column=59), migrate_from=None), SchemaColumn(name='note', type='VARCHAR', nullable=None, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/orders.sql'), line=4, column=40, end_line=4, end_column=44), migrate_from=None)), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql=\"SELECT 1 AS order_id, 'PLACED' AS status, 'x' AS note\", expanded_sql=\"SELECT 1 AS order_id, 'PLACED' AS status, 'x' AS note\", passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/staging/hidden.sql": { + "config": "CompileModelConfig(values={}, model_header_keys=('description', 'model_schema'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='hidden'), declaration=DeclarationIdentity(kind=, name='marts_shape', owner=None), kind=, through=None, enum_member=None),)", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='hidden', model_schema='marts_shape', description='Out of reach.', type_enforcement=True, meta={}, columns=(SchemaColumn(name='order_id', type='INTEGER', nullable=None, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/marts/_sqlbuild/schemas/local.sql'), line=3, column=12, end_line=3, end_column=20), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "True" + } + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/opt_out.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/opt_out.json new file mode 100644 index 0000000000..793b802804 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/opt_out.json @@ -0,0 +1,80 @@ +{ + "diagnostics": [ + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'header_opt_out' has no description\", resource_type=, resource_name='header_opt_out', column_name=None, path=PosixPath('models/header_opt_out.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/header_opt_out.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'header_opt_out\\', add this to the MODEL header in models/header_opt_out.sql:\\n MODEL (\\n description \"What one row of header_opt_out represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'invalid_call' has no description\", resource_type=, resource_name='invalid_call', column_name=None, path=PosixPath('models/invalid_call.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/invalid_call.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'invalid_call\\', add this to the MODEL header in models/invalid_call.sql:\\n MODEL (\\n description \"What one row of invalid_call represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'path_opt_out' has no description\", resource_type=, resource_name='path_opt_out', column_name=None, path=PosixPath('models/legacy/path_opt_out.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/legacy/path_opt_out.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'path_opt_out\\', add this to the MODEL header in models/legacy/path_opt_out.sql:\\n MODEL (\\n description \"What one row of path_opt_out represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'unparsed' has no description\", resource_type=, resource_name='unparsed', column_name=None, path=PosixPath('models/unparsed.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/unparsed.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'unparsed\\', add this to the MODEL header in models/unparsed.sql:\\n MODEL (\\n description \"What one row of unparsed represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P012', message='__ref(order_name) is not a valid __ref() call', resource_type=, resource_name='invalid_call', column_name=None, path=PosixPath('models/invalid_call.sql'), line=2, column=15, location=SourceLocation(path=PosixPath('models/invalid_call.sql'), line=2, column=15, end_line=2, end_column=32), related_locations=(), help='__ref() takes exactly one double-quoted name, with no comments or extra spaces inside the parentheses: __ref(\"order_name\")', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P012', message=\"__table_fn('customer_orders') is not a valid __table_fn() call\", resource_type=, resource_name='invalid_call', column_name=None, path=PosixPath('models/invalid_call.sql'), line=3, column=12, location=SourceLocation(path=PosixPath('models/invalid_call.sql'), line=3, column=12, end_line=3, end_column=41), related_locations=(), help='__table_fn() takes exactly one double-quoted name, with no comments or extra spaces inside the parentheses: __table_fn(\"customer_orders\")(...)', notes=(), affected_rules=())" + ], + "models": { + "models/header_opt_out.sql": { + "config": "CompileModelConfig(values={'sql_analysis': False}, model_header_keys=('sql_analysis',), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "SourceLocation(path=PosixPath('models/header_opt_out.sql'), line=2, column=3, end_line=None, end_column=None)", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/invalid_call.sql": { + "config": "CompileModelConfig(values={}, model_header_keys=(), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql=\"SELECT * FROM __ref(order_name)\\nCROSS JOIN __table_fn('customer_orders')\", expanded_sql=\"SELECT * FROM __ref(order_name)\\nCROSS JOIN __table_fn('customer_orders')\", passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "\"SELECT * FROM __ref(order_name)\\nCROSS JOIN __table_fn('customer_orders')\"", + "macro_usages": "()", + "query_sql": "\"SELECT * FROM __ref(order_name)\\nCROSS JOIN __table_fn('customer_orders')\"", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql=\"SELECT * FROM __ref(order_name)\\nCROSS JOIN __table_fn('customer_orders')\", expanded_sql=\"SELECT * FROM __ref(order_name)\\nCROSS JOIN __table_fn('customer_orders')\", passes=((), ()))", + "sql_validation_enabled": "False" + }, + "models/legacy/path_opt_out.sql": { + "config": "CompileModelConfig(values={'sql_analysis': False}, model_header_keys=(), matched_path_default='legacy', logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "SourceLocation(path=PosixPath('sqlbuild_project.toml'), line=8, column=1, end_line=None, end_column=None)", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/unparsed.sql": { + "config": "CompileModelConfig(values={'sql_analysis': False}, model_header_keys=('sql_analysis',), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT FROM WHERE (', expanded_sql='SELECT FROM WHERE (', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT FROM WHERE ('", + "macro_usages": "()", + "query_sql": "'SELECT FROM WHERE ('", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT FROM WHERE (', expanded_sql='SELECT FROM WHERE (', passes=((), ()))", + "sql_validation_enabled": "False" + } + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/parity.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/parity.json new file mode 100644 index 0000000000..11a257d31b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/parity.json @@ -0,0 +1,164 @@ +{ + "diagnostics": [ + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'described_columns' has no description\", resource_type=, resource_name='described_columns', column_name=None, path=PosixPath('models/marts/described_columns.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/marts/described_columns.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'described_columns\\', add this to the MODEL header in models/marts/described_columns.sql:\\n MODEL (\\n description \"What one row of described_columns represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'migrated' has no description\", resource_type=, resource_name='migrated', column_name=None, path=PosixPath('models/marts/migrated.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/marts/migrated.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'migrated\\', add this to the MODEL header in models/marts/migrated.sql:\\n MODEL (\\n description \"What one row of migrated represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'opted_out' has no description\", resource_type=, resource_name='opted_out', column_name=None, path=PosixPath('models/marts/opted_out.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/marts/opted_out.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'opted_out\\', add this to the MODEL header in models/marts/opted_out.sql:\\n MODEL (\\n description \"What one row of opted_out represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'statuses' has no description\", resource_type=, resource_name='statuses', column_name=None, path=PosixPath('models/marts/statuses.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/marts/statuses.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'statuses\\', add this to the MODEL header in models/marts/statuses.sql:\\n MODEL (\\n description \"What one row of statuses represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'stg_payments' has no description\", resource_type=, resource_name='stg_payments', column_name=None, path=PosixPath('models/staging/stg_payments.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/staging/stg_payments.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'stg_payments\\', add this to the MODEL header in models/staging/stg_payments.sql:\\n MODEL (\\n description \"What one row of stg_payments represents\",\\n ...\\n );', notes=(), affected_rules=())" + ], + "models": { + "models/marts/described_columns.sql": { + "config": "CompileModelConfig(values={'materialized': 'table'}, model_header_keys=('columns',), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql=\"SELECT 1 AS order_id, 'x' AS note\", expanded_sql=\"SELECT 1 AS order_id, 'x' AS note\", passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "\"SELECT 1 AS order_id, 'x' AS note\"", + "macro_usages": "()", + "query_sql": "\"SELECT 1 AS order_id, 'x' AS note\"", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='described_columns', model_schema=None, description=None, type_enforcement=True, meta={}, columns=(SchemaColumn(name='order_id', type='INTEGER', nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None)), location=SourceLocation(path=PosixPath('models/marts/described_columns.sql'), line=2, column=12, end_line=2, end_column=20), migrate_from=None), SchemaColumn(name='note', type=None, nullable=None, description='Free text.', meta={}, audits=(), location=SourceLocation(path=PosixPath('models/marts/described_columns.sql'), line=2, column=80, end_line=2, end_column=84), migrate_from=None)), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql=\"SELECT 1 AS order_id, 'x' AS note\", expanded_sql=\"SELECT 1 AS order_id, 'x' AS note\", passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/marts/hooked.sql": { + "config": "CompileModelConfig(values={'materialized': 'table', 'pre_hooks': [SqlHookEntry(statement='SELECT eu AS region', name=None, relative_path=None, definition_sql=None, kwargs=None, description=None, reads=())], 'post_hooks': [SqlHookEntry(statement='SELECT 2 * 100 AS cents', name=None, relative_path=None, definition_sql=None, kwargs=None, description=None, reads=())]}, model_header_keys=('description', 'post_hooks', 'pre_hooks'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='hooked'), declaration=DeclarationIdentity(kind=, name='cents', owner=None), kind=, through=None, enum_member=None),)", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='hooked', model_schema=None, description='Hooked orders.', type_enforcement=None, meta={}, columns=(), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/marts/migrated.sql": { + "config": "CompileModelConfig(values={'materialized': 'incremental', 'incremental_strategy': 'append'}, model_header_keys=('columns', 'incremental_strategy', 'materialized'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='migrated', model_schema=None, description=None, type_enforcement=None, meta={}, columns=(SchemaColumn(name='order_id', type=None, nullable=None, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/marts/migrated.sql'), line=1, column=72, end_line=1, end_column=80), migrate_from='id'),), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/marts/opted_out.sql": { + "config": "CompileModelConfig(values={'materialized': 'table', 'sql_analysis': False}, model_header_keys=('sql_analysis',), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "False" + }, + "models/marts/orders.sql": { + "config": "CompileModelConfig(values={'materialized': 'incremental', 'incremental_strategy': 'append', 'cursor': 'order_id', 'cursor_type': 'integer'}, model_header_keys=('cursor', 'cursor_type', 'description', 'incremental_strategy', 'materialized'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()', expanded_sql='SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()'", + "macro_usages": "()", + "query_sql": "'SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()'", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='orders', model_schema=None, description='Orders.', type_enforcement=None, meta={}, columns=(), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()', expanded_sql='SELECT order_id FROM __ref(\"stg_orders\")\\nWHERE order_id >= __cursor_start()', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/marts/priced.sql": { + "config": "CompileModelConfig(values={'materialized': 'table'}, model_header_keys=('columns',), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT @cents(\\'3\\') AS cents, order_id FROM __ref(\"stg_orders\")', expanded_sql='SELECT 3 * 100 AS cents, order_id FROM __ref(\"stg_orders\")', passes=((), (), (ExpansionSpan(source_start=7, source_end=18, output_start=7, output_end=14),)))", + "macro_deps": "('cents',)", + "macro_source_sql": "'SELECT @cents(\\'3\\') AS cents, order_id FROM __ref(\"stg_orders\")'", + "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='priced'), declaration=DeclarationIdentity(kind=, name='cents', owner=None), kind=, through=None, enum_member=None),)", + "query_sql": "'SELECT 3 * 100 AS cents, order_id FROM __ref(\"stg_orders\")'", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='priced', model_schema=None, description='Priced orders.', type_enforcement=None, meta={}, columns=(), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT @cents(\\'3\\') AS cents, order_id FROM __ref(\"stg_orders\")', expanded_sql='SELECT 3 * 100 AS cents, order_id FROM __ref(\"stg_orders\")', passes=((), (ExpansionSpan(source_start=7, source_end=18, output_start=7, output_end=14),)))", + "sql_validation_enabled": "True" + }, + "models/marts/statuses.sql": { + "config": "CompileModelConfig(values={'materialized': 'table'}, model_header_keys=('columns',), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='statuses'), declaration=DeclarationIdentity(kind=, name='order_status', owner=None), kind=, through=None, enum_member='PLACED'), UsageRecord(consumer=ResourceIdentity(kind=, name='statuses'), declaration=DeclarationIdentity(kind=, name='order_cap', owner=None), kind=, through=None, enum_member=None), UsageRecord(consumer=ResourceIdentity(kind=, name='statuses'), declaration=DeclarationIdentity(kind=, name='held_codes', owner=None), kind=, through=None, enum_member=None))", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT @enum(\"order_status\").PLACED AS status, @const(\"order_cap\") AS cap\\nWHERE 30 NOT IN @const(\"held_codes\")', expanded_sql=\"SELECT 'PLACED' AS status, 3 AS cap\\nWHERE 30 NOT IN (30, 31)\", passes=((), (ExpansionSpan(source_start=7, source_end=35, output_start=7, output_end=15), ExpansionSpan(source_start=47, source_end=66, output_start=27, output_end=28), ExpansionSpan(source_start=90, source_end=110, output_start=52, output_end=60)), ()))", + "macro_deps": "()", + "macro_source_sql": "\"SELECT 'PLACED' AS status, 3 AS cap\\nWHERE 30 NOT IN (30, 31)\"", + "macro_usages": "()", + "query_sql": "\"SELECT 'PLACED' AS status, 3 AS cap\\nWHERE 30 NOT IN (30, 31)\"", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='statuses', model_schema=None, description=None, type_enforcement=True, meta={}, columns=(SchemaColumn(name='status', type='VARCHAR', nullable=None, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/marts/statuses.sql'), line=1, column=17, end_line=1, end_column=23), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT @enum(\"order_status\").PLACED AS status, @const(\"order_cap\") AS cap\\nWHERE 30 NOT IN @const(\"held_codes\")', expanded_sql=\"SELECT 'PLACED' AS status, 3 AS cap\\nWHERE 30 NOT IN (30, 31)\", passes=((ExpansionSpan(source_start=7, source_end=35, output_start=7, output_end=15), ExpansionSpan(source_start=47, source_end=66, output_start=27, output_end=28), ExpansionSpan(source_start=90, source_end=110, output_start=52, output_end=60)), ()))", + "sql_validation_enabled": "True" + }, + "models/staging/stg_orders.sql": { + "config": "CompileModelConfig(values={'materialized': 'view', 'tags': ['staged', 'daily']}, model_header_keys=('description', 'tags'), matched_path_default='staging', logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql=\"SELECT 1 AS order_id, '@@region' AS region -- note\", expanded_sql=\"SELECT 1 AS order_id, 'eu' AS region -- note\", passes=((ExpansionSpan(source_start=23, source_end=31, output_start=23, output_end=25),), (), ()))", + "macro_deps": "()", + "macro_source_sql": "\"SELECT 1 AS order_id, 'eu' AS region -- note\"", + "macro_usages": "()", + "query_sql": "\"SELECT 1 AS order_id, 'eu' AS region -- note\"", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='stg_orders', model_schema=None, description='Staged orders.', type_enforcement=None, meta={}, columns=(), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "None", + "sql_validation_enabled": "True" + }, + "models/staging/stg_payments.sql": { + "config": "CompileModelConfig(values={'materialized': 'view', 'tags': ['staged', 'payments']}, model_header_keys=('tags',), matched_path_default='staging', logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT method FROM __seed(\"payment_methods\")', expanded_sql='SELECT method FROM __seed(\"payment_methods\")', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT method FROM __seed(\"payment_methods\")'", + "macro_usages": "()", + "query_sql": "'SELECT method FROM __seed(\"payment_methods\")'", + "references": "(CompileSqlReference(ref_kind=, ref_name='payment_methods', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT method FROM __seed(\"payment_methods\")', expanded_sql='SELECT method FROM __seed(\"payment_methods\")', passes=((), ()))", + "sql_validation_enabled": "True" + } + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/python_hooks.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/python_hooks.json new file mode 100644 index 0000000000..93c2e47e1e --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/python_hooks.json @@ -0,0 +1,61 @@ +{ + "diagnostics": [ + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'summary' has no description\", resource_type=, resource_name='summary', column_name=None, path=PosixPath('models/marts/summary.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/marts/summary.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'summary\\', add this to the MODEL header in models/marts/summary.sql:\\n MODEL (\\n description \"What one row of summary represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'orders' has no description\", resource_type=, resource_name='orders', column_name=None, path=PosixPath('models/orders.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/orders.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'orders\\', add this to the MODEL header in models/orders.sql:\\n MODEL (\\n description \"What one row of orders represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'hidden' has no description\", resource_type=, resource_name='hidden', column_name=None, path=PosixPath('models/staging/hidden.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/staging/hidden.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'hidden\\', add this to the MODEL header in models/staging/hidden.sql:\\n MODEL (\\n description \"What one row of hidden represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='S006', message=\"Python hook 'refresh' is not visible from 'models/staging/hidden.sql'. It is defined at models/marts/_sqlbuild/hooks/python/local.py with inherited scope owned by 'models/marts'\", resource_type=, resource_name='hidden', column_name=None, path=PosixPath('models/staging/hidden.sql'), line=None, column=None, location=None, related_locations=(), help=\"move 'models/marts/_sqlbuild/hooks/python/local.py' to a hooks/python/ role owned by 'models/staging' or one of its parent folders, or to the project-wide hooks/python/ when consumers span resource trees\", notes=(), affected_rules=())" + ], + "models": { + "models/marts/summary.sql": { + "config": "CompileModelConfig(values={'post_hooks': [PythonHookEntry(name='refresh', kwargs={}), PythonHookEntry(name='flexible', kwargs={})]}, model_header_keys=('post_hooks',), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='summary'), declaration=DeclarationIdentity(kind=, name='refresh', owner=None), kind=, through=None, enum_member=None), UsageRecord(consumer=ResourceIdentity(kind=, name='summary'), declaration=DeclarationIdentity(kind=, name='flexible', owner=None), kind=, through=None, enum_member=None))", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/orders.sql": { + "config": "CompileModelConfig(values={'pre_hooks': [PythonHookEntry(name='notify', kwargs={'channel': 'ops'})], 'post_hooks': [PythonHookEntry(name='flexible', kwargs={'any': 1}), PythonHookEntry(name='notify', kwargs={'channel': 'c', 'level': 'x'})]}, model_header_keys=('post_hooks', 'pre_hooks'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='orders'), declaration=DeclarationIdentity(kind=, name='flexible', owner=None), kind=, through=None, enum_member=None), UsageRecord(consumer=ResourceIdentity(kind=, name='orders'), declaration=DeclarationIdentity(kind=, name='notify', owner=None), kind=, through=None, enum_member=None))", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/staging/hidden.sql": { + "config": "CompileModelConfig(values={'post_hooks': [PythonHookEntry(name='refresh', kwargs={})]}, model_header_keys=('post_hooks',), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='hidden'), declaration=DeclarationIdentity(kind=, name='refresh', owner=None), kind=, through=None, enum_member=None),)", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "True" + } + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/sql_hooks.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/sql_hooks.json new file mode 100644 index 0000000000..afc0ec40b1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/sql_hooks.json @@ -0,0 +1,42 @@ +{ + "diagnostics": [ + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'loaded' has no description\", resource_type=, resource_name='loaded', column_name=None, path=PosixPath('models/marts/loaded.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/marts/loaded.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'loaded\\', add this to the MODEL header in models/marts/loaded.sql:\\n MODEL (\\n description \"What one row of loaded represents\",\\n ...\\n );', notes=(), affected_rules=())", + "CompilerDiagnostic(phase=, severity=, code='P010', message=\"model 'stg_orders' has no description\", resource_type=, resource_name='stg_orders', column_name=None, path=PosixPath('models/staging/stg_orders.sql'), line=1, column=1, location=SourceLocation(path=PosixPath('models/staging/stg_orders.sql'), line=1, column=1, end_line=None, end_column=None), related_locations=(), help='to describe model \\'stg_orders\\', add this to the MODEL header in models/staging/stg_orders.sql:\\n MODEL (\\n description \"What one row of stg_orders represents\",\\n ...\\n );', notes=(), affected_rules=())" + ], + "models": { + "models/marts/loaded.sql": { + "config": "CompileModelConfig(values={'schema': 'analytics', 'pre_hooks': [SqlHookEntry(statement=\"GRANT SELECT ON analytics.loaded TO 'reader', 'writer'\", name='grant_read', relative_path=PosixPath('hooks/sql/grant_read.sql'), definition_sql=\"GRANT SELECT ON @@CTX:destination.qualified TO @'role'\", kwargs={'role': ['reader', 'writer']}, description='Grant read access.', reads=())], 'post_hooks': [SqlHookEntry(statement='INSERT INTO audit.loads SELECT \\'PLACED\\', 3, 5 * 100, \\'eu\\' FROM __ref(\"stg_orders\")', name='log_load', relative_path=PosixPath('hooks/sql/log_load.sql'), definition_sql='INSERT INTO @target SELECT @enum(\"order_status\").PLACED, @const(\"order_cap\"), @cents(@amount), \\'@@region\\' FROM __ref(\"stg_orders\")', kwargs={'target': 'audit.loads', 'amount': 5}, description='Log a load.', reads=(SqlResourceRef(kind=, name='stg_orders'),)), SqlHookEntry(statement='SELECT 3 FROM __ref(\"stg_orders\")', name=None, relative_path=None, definition_sql=None, kwargs=None, description=None, reads=(SqlResourceRef(kind=, name='stg_orders'),))]}, model_header_keys=('post_hooks', 'pre_hooks', 'schema'), matched_path_default=None, logical_schema='analytics', layer_schema='analytics', logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='loaded'), declaration=DeclarationIdentity(kind=, name='grant_read', owner=None), kind=, through=None, enum_member=None), UsageRecord(consumer=ResourceIdentity(kind=, name='loaded'), declaration=DeclarationIdentity(kind=, name='log_load', owner=None), kind=, through=None, enum_member=None), UsageRecord(consumer=DeclarationIdentity(kind=, name='log_load', owner=None), declaration=DeclarationIdentity(kind=, name='order_status', owner=None), kind=, through=None, enum_member='PLACED'), UsageRecord(consumer=DeclarationIdentity(kind=, name='log_load', owner=None), declaration=DeclarationIdentity(kind=, name='order_cap', owner=None), kind=, through=None, enum_member=None), UsageRecord(consumer=DeclarationIdentity(kind=, name='log_load', owner=None), declaration=DeclarationIdentity(kind=, name='cents', owner=None), kind=, through=None, enum_member=None), UsageRecord(consumer=ResourceIdentity(kind=, name='loaded'), declaration=DeclarationIdentity(kind=, name='order_cap', owner=None), kind=, through=None, enum_member=None))", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT order_id FROM __ref(\"stg_orders\")', expanded_sql='SELECT order_id FROM __ref(\"stg_orders\")', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT order_id FROM __ref(\"stg_orders\")'", + "macro_usages": "()", + "query_sql": "'SELECT order_id FROM __ref(\"stg_orders\")'", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT order_id FROM __ref(\"stg_orders\")', expanded_sql='SELECT order_id FROM __ref(\"stg_orders\")', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/staging/stg_orders.sql": { + "config": "CompileModelConfig(values={}, model_header_keys=(), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT 1 AS order_id'", + "macro_usages": "()", + "query_sql": "'SELECT 1 AS order_id'", + "references": "()", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "None", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT 1 AS order_id', expanded_sql='SELECT 1 AS order_id', passes=((), ()))", + "sql_validation_enabled": "True" + } + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/expected/waffle_shop.json b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/waffle_shop.json new file mode 100644 index 0000000000..66efea9cb8 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/expected/waffle_shop.json @@ -0,0 +1,226 @@ +{ + "diagnostics": [], + "models": { + "models/intermediate/customer_status_snapshot.sql": { + "config": "CompileModelConfig(values={'materialized': 'incremental', 'incremental_strategy': 'merge', 'unique_key': ['customer_id'], 'cursor': 'last_ordered_at', 'cursor_type': 'timestamp', 'cursor_grain': 'second', 'cursor_inputs': {'fact_orders': 'ordered_at'}, 'replay_on_change': 'full', 'on_schema_change': 'sync_all_columns', 'tags': ['intermediate', 'acceptance']}, model_header_keys=('columns', 'cursor', 'cursor_grain', 'cursor_inputs', 'cursor_type', 'description', 'incremental_strategy', 'materialized', 'on_schema_change', 'replay_on_change', 'tags', 'unique_key'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id', expanded_sql='SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id'", + "macro_usages": "()", + "query_sql": "'SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id'", + "references": "(CompileSqlReference(ref_kind=, ref_name='fact_orders', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='customer_status_snapshot', model_schema=None, description='Merge-based customer snapshot with timestamp cursor and explicit backfill policies.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='customer_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None)), location=SourceLocation(path=PosixPath('models/intermediate/customer_status_snapshot.sql'), line=16, column=5, end_line=16, end_column=16), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id', expanded_sql='SELECT\\n customer_id,\\n MAX(ordered_at) AS last_ordered_at,\\n MAX(order_status) AS latest_order_status,\\n COUNT(*) AS total_orders,\\n SUM(line_total_cents) AS total_revenue_cents\\nFROM __ref(\"fact_orders\")\\nGROUP BY customer_id', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/intermediate/order_status_index.sql": { + "config": "CompileModelConfig(values={'materialized': 'incremental', 'incremental_strategy': 'delete_insert', 'cursor': 'order_id', 'cursor_type': 'integer', 'on_schema_change': 'append_new_columns', 'replay_on_change': 'bounded-7d', 'tags': ['intermediate', 'acceptance']}, model_header_keys=('columns', 'cursor', 'cursor_type', 'description', 'incremental_strategy', 'materialized', 'on_schema_change', 'replay_on_change', 'tags'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")', expanded_sql='SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")'", + "macro_usages": "()", + "query_sql": "'SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")'", + "references": "(CompileSqlReference(ref_kind=, ref_name='fact_orders', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='order_status_index', model_schema=None, description='Integer-cursor incremental projection over fact orders.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='order_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None)), location=SourceLocation(path=PosixPath('models/intermediate/order_status_index.sql'), line=11, column=5, end_line=11, end_column=13), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")', expanded_sql='SELECT\\n order_id,\\n customer_id,\\n order_status,\\n ordered_at,\\n line_total_cents\\nFROM __ref(\"fact_orders\")', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/marts/daily_activity_rollup.sql": { + "config": "CompileModelConfig(values={'materialized': 'incremental', 'incremental_strategy': 'delete_insert', 'cursor': 'activity_day', 'cursor_type': 'timestamp', 'cursor_grain': 'day', 'microbatch_strategy': 'watermark', 'cursor_watermark_mode': 'all', 'cursor_inputs': {'hourly_order_activity': {'column': 'activity_hour', 'roles': ['filter', 'watermark']}}, 'incremental_mode': 'microbatch', 'batch_size': '2d', 'replay_on_change': 'bounded-14d', 'tags': ['marts', 'acceptance']}, model_header_keys=('audits', 'batch_size', 'columns', 'cursor', 'cursor_grain', 'cursor_inputs', 'cursor_type', 'cursor_watermark_mode', 'description', 'incremental_mode', 'incremental_strategy', 'materialized', 'microbatch_strategy', 'replay_on_change', 'tags'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n @timestamp_trunc(\\'day\\', \\'activity_hour\\') AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY @timestamp_trunc(\\'day\\', \\'activity_hour\\')', expanded_sql='SELECT\\n DATE_TRUNC(\\'day\\', activity_hour) AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY DATE_TRUNC(\\'day\\', activity_hour)', passes=((), (), (ExpansionSpan(source_start=9, source_end=49, output_start=9, output_end=41), ExpansionSpan(source_start=232, source_end=272, output_start=224, output_end=256))))", + "macro_deps": "('timestamp_trunc',)", + "macro_source_sql": "'SELECT\\n @timestamp_trunc(\\'day\\', \\'activity_hour\\') AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY @timestamp_trunc(\\'day\\', \\'activity_hour\\')'", + "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='daily_activity_rollup'), declaration=DeclarationIdentity(kind=, name='timestamp_trunc', owner=None), kind=, through=None, enum_member=None),)", + "query_sql": "'SELECT\\n DATE_TRUNC(\\'day\\', activity_hour) AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY DATE_TRUNC(\\'day\\', activity_hour)'", + "references": "(CompileSqlReference(ref_kind=, ref_name='hourly_order_activity', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='daily_activity_rollup', model_schema=None, description='Downstream daily microbatch rollup with a wider batch size than its hourly upstream.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='activity_day', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('models/marts/daily_activity_rollup.sql'), line=18, column=5, end_line=18, end_column=17), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'orders_placed >= 0'}, name='daily_orders_placed_is_non_negative', description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n @timestamp_trunc(\\'day\\', \\'activity_hour\\') AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY @timestamp_trunc(\\'day\\', \\'activity_hour\\')', expanded_sql='SELECT\\n DATE_TRUNC(\\'day\\', activity_hour) AS activity_day,\\n SUM(orders_placed) AS orders_placed,\\n SUM(waffles_ordered) AS waffles_ordered,\\n SUM(revenue_cents) AS revenue_cents\\nFROM __ref(\"hourly_order_activity\")\\nGROUP BY DATE_TRUNC(\\'day\\', activity_hour)', passes=((), (ExpansionSpan(source_start=9, source_end=49, output_start=9, output_end=41), ExpansionSpan(source_start=232, source_end=272, output_start=224, output_end=256))))", + "sql_validation_enabled": "True" + }, + "models/marts/daily_order_partitioned.sql": { + "config": "CompileModelConfig(values={'materialized': 'partition_tracked', 'tags': ['marts'], 'placeholders': {'partition_start': \"'2026-04-01'\", 'partition_end': \"'2026-04-05'\"}, 'config': {'tracking_table': 'partition_state', 'partition_column': 'order_date', 'date_range_start': '2026-04-01', 'date_range_end': '2026-04-05'}}, model_header_keys=('audits', 'columns', 'config', 'description', 'materialized', 'placeholders', 'tags'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)', expanded_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)', passes=((ExpansionSpan(source_start=254, source_end=272, output_start=254, output_end=272), ExpansionSpan(source_start=322, source_end=338, output_start=322, output_end=338)), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)'", + "macro_usages": "()", + "query_sql": "'SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)'", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='daily_order_partitioned', model_schema=None, description='Partition-tracked daily order summary using custom materialization.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='order_date', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('models/marts/daily_order_partitioned.sql'), line=16, column=5, end_line=16, end_column=15), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'waffles_ordered > 0'}, name='waffles_ordered_is_positive', description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)', expanded_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS order_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_ordered,\\n COUNT(DISTINCT o.customer_id) AS unique_customers\\nFROM __ref(\"stg_orders\") o\\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\\nGROUP BY CAST(o.ordered_at AS DATE)', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/marts/daily_revenue.sql": { + "config": "CompileModelConfig(values={'materialized': 'table', 'tags': ['marts']}, model_header_keys=('audits', 'columns', 'description', 'materialized', 'tags'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)', expanded_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)'", + "macro_usages": "()", + "query_sql": "'SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)'", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None))", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='daily_revenue', model_schema=None, description='Daily revenue aggregation.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='revenue_date', type=None, nullable=False, description=None, meta={}, audits=(), location=SourceLocation(path=PosixPath('models/marts/daily_revenue.sql'), line=12, column=5, end_line=12, end_column=17), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'total_revenue_cents >= 0'}, name='revenue_is_non_negative', description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)', expanded_sql='SELECT\\n CAST(o.ordered_at AS DATE) AS revenue_date,\\n COUNT(DISTINCT o.order_id) AS order_count,\\n SUM(o.quantity) AS waffles_sold,\\n SUM(p.amount_cents) AS total_revenue_cents,\\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY CAST(o.ordered_at AS DATE)', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/marts/dim_customers.sql": { + "config": "CompileModelConfig(values={'materialized': 'table', 'tags': ['marts']}, model_header_keys=('columns', 'description', 'materialized', 'tags'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at', expanded_sql='SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at'", + "macro_usages": "()", + "query_sql": "'SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at'", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_customers', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None))", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='dim_customers', model_schema=None, description='Customer dimension with lifetime metrics.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='customer_id', type=None, nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None)), location=SourceLocation(path=PosixPath('models/marts/dim_customers.sql'), line=6, column=5, end_line=6, end_column=16), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at', expanded_sql='SELECT\\n c.customer_id,\\n c.first_name,\\n c.last_name,\\n c.email,\\n c.created_at AS customer_since,\\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\\nFROM __ref(\"stg_customers\") c\\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = \\'success\\'\\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/marts/fact_orders.sql": { + "config": "CompileModelConfig(values={'materialized': 'table', 'tags': ['marts']}, model_header_keys=('columns', 'description', 'materialized', 'tags'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n @line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', expanded_sql='SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n w.price_cents * o.quantity AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', passes=((), (), (ExpansionSpan(source_start=124, source_end=172, output_start=124, output_end=150),)))", + "macro_deps": "('line_total_cents',)", + "macro_source_sql": "'SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n @line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id'", + "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='fact_orders'), declaration=DeclarationIdentity(kind=, name='line_total_cents', owner=None), kind=, through=None, enum_member=None),)", + "query_sql": "'SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n w.price_cents * o.quantity AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id'", + "references": "(CompileSqlReference(ref_kind=, ref_name='is_completed_order', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='is_completed_order_py', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='waffle_types', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None))", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='fact_orders', model_schema=None, description='Order fact table with waffle and payment details.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='order_id', type=None, nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('models/marts/fact_orders.sql'), line=6, column=5, end_line=6, end_column=13), migrate_from=None),), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n @line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', expanded_sql='SELECT\\n o.order_id,\\n o.customer_id,\\n o.waffle_type_id,\\n w.waffle_name,\\n w.category AS waffle_category,\\n o.quantity,\\n w.price_cents * o.quantity AS line_total_cents,\\n o.ordered_at,\\n o.status AS order_status,\\n __udf(\"is_completed_order\")(o.status) AS is_completed_order,\\n __udf(\"is_completed_order_py\")(o.status) AS is_completed_order_py,\\n p.payment_method,\\n p.payment_status,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', passes=((), (ExpansionSpan(source_start=124, source_end=172, output_start=124, output_end=150),)))", + "sql_validation_enabled": "True" + }, + "models/marts/hourly_activity_with_daily_context.sql": { + "config": "CompileModelConfig(values={'materialized': 'incremental', 'incremental_strategy': 'delete_insert', 'cursor': 'activity_hour', 'cursor_type': 'timestamp', 'cursor_grain': 'hour', 'microbatch_strategy': 'watermark', 'cursor_watermark_mode': 'all', 'cursor_inputs': {'daily_activity_rollup': {'column': 'activity_day', 'roles': ['filter', 'watermark']}}, 'incremental_mode': 'microbatch', 'batch_size': '6h', 'tags': ['marts', 'acceptance']}, model_header_keys=('audits', 'batch_size', 'columns', 'cursor', 'cursor_grain', 'cursor_inputs', 'cursor_type', 'cursor_watermark_mode', 'description', 'incremental_mode', 'incremental_strategy', 'materialized', 'microbatch_strategy', 'tags'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON @timestamp_trunc(\\'day\\', \\'h.activity_hour\\') = d.activity_day', expanded_sql='SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON DATE_TRUNC(\\'day\\', h.activity_hour) = d.activity_day', passes=((), (), (ExpansionSpan(source_start=211, source_end=253, output_start=211, output_end=245),)))", + "macro_deps": "('timestamp_trunc',)", + "macro_source_sql": "'SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON @timestamp_trunc(\\'day\\', \\'h.activity_hour\\') = d.activity_day'", + "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='hourly_activity_with_daily_context'), declaration=DeclarationIdentity(kind=, name='timestamp_trunc', owner=None), kind=, through=None, enum_member=None),)", + "query_sql": "'SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON DATE_TRUNC(\\'day\\', h.activity_hour) = d.activity_day'", + "references": "(CompileSqlReference(ref_kind=, ref_name='hourly_order_activity', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='daily_activity_rollup', ref_package=None, call_argument_count=None))", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='hourly_activity_with_daily_context', model_schema=None, description='Hourly downstream microbatch model that depends on a coarser-grain daily upstream.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='activity_hour', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('models/marts/hourly_activity_with_daily_context.sql'), line=17, column=5, end_line=17, end_column=18), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'day_orders_placed >= orders_placed'}, name='day_orders_cover_hourly_orders', description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON @timestamp_trunc(\\'day\\', \\'h.activity_hour\\') = d.activity_day', expanded_sql='SELECT\\n h.activity_hour,\\n h.orders_placed,\\n d.orders_placed AS day_orders_placed,\\n h.waffles_ordered,\\n h.revenue_cents\\nFROM __ref(\"hourly_order_activity\") h\\nINNER JOIN __ref(\"daily_activity_rollup\") d\\n ON DATE_TRUNC(\\'day\\', h.activity_hour) = d.activity_day', passes=((), (ExpansionSpan(source_start=211, source_end=253, output_start=211, output_end=245),)))", + "sql_validation_enabled": "True" + }, + "models/marts/hourly_order_activity.sql": { + "config": "CompileModelConfig(values={'materialized': 'incremental', 'incremental_strategy': 'delete_insert', 'cursor': 'activity_hour', 'cursor_type': 'timestamp', 'cursor_grain': 'hour', 'microbatch_strategy': 'watermark', 'cursor_watermark_mode': 'all', 'cursor_inputs': {'fact_orders': {'column': 'ordered_at', 'roles': ['filter', 'watermark']}}, 'incremental_mode': 'microbatch', 'batch_size': '1d', 'tags': ['marts']}, model_header_keys=('audits', 'batch_size', 'columns', 'cursor', 'cursor_grain', 'cursor_inputs', 'cursor_type', 'cursor_watermark_mode', 'description', 'incremental_mode', 'incremental_strategy', 'materialized', 'microbatch_strategy', 'tags'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n @timestamp_trunc(\\'hour\\', \\'o.ordered_at\\') AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY @timestamp_trunc(\\'hour\\', \\'o.ordered_at\\')', expanded_sql='SELECT\\n DATE_TRUNC(\\'hour\\', o.ordered_at) AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY DATE_TRUNC(\\'hour\\', o.ordered_at)', passes=((), (), (ExpansionSpan(source_start=9, source_end=49, output_start=9, output_end=41), ExpansionSpan(source_start=215, source_end=255, output_start=207, output_end=239))))", + "macro_deps": "('timestamp_trunc',)", + "macro_source_sql": "'SELECT\\n @timestamp_trunc(\\'hour\\', \\'o.ordered_at\\') AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY @timestamp_trunc(\\'hour\\', \\'o.ordered_at\\')'", + "macro_usages": "(UsageRecord(consumer=ResourceIdentity(kind=, name='hourly_order_activity'), declaration=DeclarationIdentity(kind=, name='timestamp_trunc', owner=None), kind=, through=None, enum_member=None),)", + "query_sql": "'SELECT\\n DATE_TRUNC(\\'hour\\', o.ordered_at) AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY DATE_TRUNC(\\'hour\\', o.ordered_at)'", + "references": "(CompileSqlReference(ref_kind=, ref_name='fact_orders', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='hourly_order_activity', model_schema=None, description='Hourly order activity aggregated via microbatch incremental.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='activity_hour', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('models/marts/hourly_order_activity.sql'), line=17, column=5, end_line=17, end_column=18), migrate_from=None),), dynamic_columns=(), audits=(SchemaAuditInstance(definition_name='expression_is_true', arguments={'expression': 'orders_placed >= 0'}, name='orders_placed_is_non_negative', description=None, severity=None, run_scope='delta_and_final', always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n @timestamp_trunc(\\'hour\\', \\'o.ordered_at\\') AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY @timestamp_trunc(\\'hour\\', \\'o.ordered_at\\')', expanded_sql='SELECT\\n DATE_TRUNC(\\'hour\\', o.ordered_at) AS activity_hour,\\n COUNT(*) AS orders_placed,\\n SUM(o.quantity) AS waffles_ordered,\\n SUM(o.line_total_cents) AS revenue_cents\\nFROM __ref(\"fact_orders\") o\\nGROUP BY DATE_TRUNC(\\'hour\\', o.ordered_at)', passes=((), (ExpansionSpan(source_start=9, source_end=49, output_start=9, output_end=41), ExpansionSpan(source_start=215, source_end=255, output_start=207, output_end=239))))", + "sql_validation_enabled": "True" + }, + "models/marts/scenario_order_prices.sql": { + "config": "CompileModelConfig(values={'materialized': 'table', 'tags': ['marts', 'scenario']}, model_header_keys=('description', 'materialized', 'tags'), matched_path_default=None, logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', expanded_sql='SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id'", + "macro_usages": "()", + "query_sql": "'SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id'", + "references": "(CompileSqlReference(ref_kind=, ref_name='stg_orders', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='waffle_types', ref_package=None, call_argument_count=None), CompileSqlReference(ref_kind=, ref_name='stg_payments', ref_package=None, call_argument_count=None))", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='scenario_order_prices', model_schema=None, description='Small model used by scenario CLI e2e retain coverage.', type_enforcement=None, meta={}, columns=(), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', expanded_sql='SELECT\\n o.order_id,\\n o.quantity,\\n w.waffle_name,\\n w.price_cents,\\n w.price_cents * o.quantity AS line_total_cents,\\n p.amount_cents AS payment_amount_cents\\nFROM __ref(\"stg_orders\") o\\nINNER JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/staging/stg_customers.sql": { + "config": "CompileModelConfig(values={'materialized': 'view', 'tags': ['staging']}, model_header_keys=('columns', 'description', 'materialized', 'tags'), matched_path_default='staging', logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")', expanded_sql='SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")'", + "macro_usages": "()", + "query_sql": "'SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")'", + "references": "(CompileSqlReference(ref_kind=, ref_name='raw_customers', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='stg_customers', model_schema=None, description='Cleaned customer records.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='customer_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None)), location=SourceLocation(path=PosixPath('models/staging/stg_customers.sql'), line=6, column=5, end_line=6, end_column=16), migrate_from=None), SchemaColumn(name='email', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('models/staging/stg_customers.sql'), line=7, column=5, end_line=7, end_column=10), migrate_from=None)), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")', expanded_sql='SELECT\\n id AS customer_id,\\n first_name,\\n last_name,\\n email,\\n created_at\\nFROM __source(\"raw_customers\")', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/staging/stg_orders.sql": { + "config": "CompileModelConfig(values={'materialized': 'view', 'tags': ['staging']}, model_header_keys=('columns', 'description', 'materialized', 'tags'), matched_path_default='staging', logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")', expanded_sql='SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")'", + "macro_usages": "()", + "query_sql": "'SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")'", + "references": "(CompileSqlReference(ref_kind=, ref_name='raw_orders', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='stg_orders', model_schema=None, description='Cleaned order records.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='order_id', type=None, nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None)), location=SourceLocation(path=PosixPath('models/staging/stg_orders.sql'), line=6, column=5, end_line=6, end_column=13), migrate_from=None), SchemaColumn(name='customer_id', type=None, nullable=False, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('models/staging/stg_orders.sql'), line=7, column=5, end_line=7, end_column=16), migrate_from=None), SchemaColumn(name='waffle_type_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='relationships', arguments={'to': '__seed(\"waffle_types\")', 'field': 'waffle_type_id'}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('models/staging/stg_orders.sql'), line=8, column=5, end_line=8, end_column=19), migrate_from=None), SchemaColumn(name='status', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='accepted_values', arguments={'values': ['placed', 'preparing', 'ready', 'completed', 'cancelled']}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None),), location=SourceLocation(path=PosixPath('models/staging/stg_orders.sql'), line=11, column=5, end_line=11, end_column=11), migrate_from=None)), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")', expanded_sql='SELECT\\n id AS order_id,\\n customer_id,\\n waffle_type_id,\\n quantity,\\n ordered_at,\\n status\\nFROM __source(\"raw_orders\")', passes=((), ()))", + "sql_validation_enabled": "True" + }, + "models/staging/stg_payments.sql": { + "config": "CompileModelConfig(values={'materialized': 'view', 'tags': ['staging']}, model_header_keys=('columns', 'description', 'materialized', 'tags'), matched_path_default='staging', logical_schema=None, layer_schema=None, logical_database=None, time_travel_retention=ResolvedTimeTravelRetention(desired_days=None, unmanaged=True, source=None), table_type=ResolvedTableType(value=, source=, declared=False))", + "constant_declarations": "()", + "declaration_usages": "()", + "enum_columns": "{}", + "enum_declarations": "()", + "lint_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")', expanded_sql='SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")', passes=((), (), ()))", + "macro_deps": "()", + "macro_source_sql": "'SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")'", + "macro_usages": "()", + "query_sql": "'SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")'", + "references": "(CompileSqlReference(ref_kind=, ref_name='raw_payments', ref_package=None, call_argument_count=None),)", + "rejected_sql_analysis_opt_out": "None", + "schema_entry": "SchemaModelEntry(name='stg_payments', model_schema=None, description='Cleaned payment records.', type_enforcement=None, meta={}, columns=(SchemaColumn(name='payment_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None), SchemaAuditInstance(definition_name='unique', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None)), location=SourceLocation(path=PosixPath('models/staging/stg_payments.sql'), line=6, column=5, end_line=6, end_column=15), migrate_from=None), SchemaColumn(name='order_id', type=None, nullable=None, description=None, meta={}, audits=(SchemaAuditInstance(definition_name='not_null', arguments={}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None), SchemaAuditInstance(definition_name='relationships', arguments={'to': '__source(\"raw_orders\")', 'field': 'id'}, name=None, description=None, severity=None, run_scope=None, always_run=False, thresholds=None, minimum_samples=None, evidence_limit=None)), location=SourceLocation(path=PosixPath('models/staging/stg_payments.sql'), line=7, column=5, end_line=7, end_column=13), migrate_from=None)), dynamic_columns=(), audits=(), tags=())", + "sql_expansion": "CompiledSqlExpansion(authored_sql='SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")', expanded_sql='SELECT\\n id AS payment_id,\\n order_id,\\n amount_cents,\\n payment_method,\\n paid_at,\\n status AS payment_status\\nFROM __source(\"raw_payments\")', passes=((), ()))", + "sql_validation_enabled": "True" + } + } +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py new file mode 100644 index 0000000000..8d44ae5476 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -0,0 +1,717 @@ +"""Helpers for native compiled project integration tests.""" + +from __future__ import annotations + +import json +import shutil +from collections import Counter +from collections.abc import Callable, Iterator +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any + +import pytest + +import sqlbuild._native as _native +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds +from sqlbuild.compiler.compile import models as compile_models +from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project +from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core +from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references +from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild.compiler.compile.models import ( + CompileAdapterContext, + CompileModelConfig, + CompileModelInput, + CompileProjectInputs, + CompilerDiagnostic, +) +from sqlbuild.compiler.compile.types import ( + CompiledAuditView, + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompileInputsView, +) +from sqlbuild.compiler.compiled_project.main import compiled_project_facts as facts_module +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile +from sqlbuild.compiler.pipeline.main.project import compile_project +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry +from sqlbuild.sql_values.types import CollectionRendering +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + +TYPE_PROOF_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' + '[defaults]\ncontract = "enforced"\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.',\n" + " columns (order_id (type INTEGER), amount (type INTEGER)),\n" + ");\n\n" + "SELECT CAST(1 AS INTEGER) AS order_id, CAST(5 AS INTEGER) AS amount\n" + ), + "models/orders.sql": ( + "MODEL (description 'Typed orders.',\n" + " columns (order_id (type INTEGER), amount (type BIGINT)),\n" + ");\n\n" + 'SELECT order_id, amount\nFROM __ref("stg_orders")\n' + ), +} + + +def forbid_hand_built_projects(*, monkeypatch: pytest.MonkeyPatch) -> None: + """Fail if a consumer records a project from `CompiledProject` instead of the compile's.""" + + def refuse(project: object) -> object: + raise AssertionError(f"rules rebuilt project facts from {type(project).__name__}") + + monkeypatch.setattr(facts_module, "hand_built_project_impl", refuse) + + +def skip_model_retention(*, monkeypatch: pytest.MonkeyPatch) -> None: + """Run the native model loop into a throwaway project instead of the compile's own. + + The compile's own project still runs the loop over no models, so it keeps the loop that + attaches its other resources. + """ + + native_loop: Callable[..., object] = attachment_core._run_native_model_loop + + def discarded(**kwargs: Any) -> object: + loop: Any = kwargs["loop"] + _ = native_loop( + **{ + **kwargs, + "loop": replace( + loop, discovered_inputs=replace(loop.discovered_inputs, model_files=()) + ), + } + ) + kwargs["native_project"] = _native.NativeCompiledProject() + return native_loop(**kwargs) + + monkeypatch.setattr(attachment_core, "_run_native_model_loop", discarded) + + +LEGACY_PROJECT_TYPES: tuple[str, ...] = ( + "CompileProjectInputs", + "CompileModelInput", + "CompiledProject", + "CompiledModel", +) + + +def count_legacy_constructions(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: + """Count every construction of the Python project representation, `replace` included.""" + + counts: Counter[str] = Counter() + for name in LEGACY_PROJECT_TYPES: + model_type: type = getattr(compile_models, name) + monkeypatch.setattr( + model_type, "__init__", _counted_init(counts=counts, name=name, model_type=model_type) + ) + return counts + + +def _counted_init(*, counts: Counter[str], name: str, model_type: type) -> object: + original: Callable[..., None] = model_type.__init__ + + def counted(self: object, *args: object, **kwargs: object) -> None: + counts[name] += 1 + original(self, *args, **kwargs) + + return counted + + +PARITY_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nregion = "eu"\n\n' + '[defaults]\nmaterialized = "table"\n\n[path_defaults.staging]\n' + 'materialized = "view"\ntags = ["staged"]\n' + ), + "seeds/payment_methods.csv": "method\ncard\n", + "seeds/payment_methods.yml": ( + "seeds:\n - name: payment_methods\n description: Payment methods.\n" + " columns:\n - name: method\n type: VARCHAR\n" + ), + "models/staging/stg_orders.sql": ( + "MODEL (description 'Staged orders.', tags [daily]);\n" + "SELECT 1 AS order_id, '@@region' AS region -- note\n" + ), + "models/staging/stg_payments.sql": ( + 'MODEL (tags [payments]);\nSELECT method FROM __seed("payment_methods")\n' + ), + "models/marts/orders.sql": ( + "MODEL (description 'Orders.', materialized incremental, incremental_strategy append," + " cursor order_id, cursor_type integer);\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + "WHERE order_id >= __cursor_start()\n" + ), + "models/marts/opted_out.sql": ("MODEL (sql_analysis false);\nSELECT 1 AS order_id\n"), + "models/marts/described_columns.sql": ( + "MODEL (\n columns (order_id (type INTEGER, nullable false," + ' audits [not_null, unique]), note (description "Free text.")),\n' + ");\n" + "SELECT 1 AS order_id, 'x' AS note\n" + ), + "enums/status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + "constants/limits.sql": ( + "CONSTANT (name order_cap, value 3);\nCONSTANT (name held_codes, value [30, 31]);\n" + ), + "models/marts/statuses.sql": ( + "MODEL (columns (status (type VARCHAR)));\n" + 'SELECT @enum("order_status").PLACED AS status, @const("order_cap") AS cap\n' + 'WHERE 30 NOT IN @const("held_codes")\n' + ), + "macros/money.py": 'def cents(value: str) -> str:\n return f"{value} * 100"\n', + "models/marts/priced.sql": ( + "MODEL (description 'Priced orders.');\n" + "SELECT @cents('3') AS cents, order_id FROM __ref(\"stg_orders\")\n" + ), + "models/marts/hooked.sql": ( + "MODEL (\n description 'Hooked orders.',\n" + " pre_hooks [inline_sql('SELECT @@region AS region')],\n" + " post_hooks [inline_sql('SELECT @cents(2) AS cents')],\n);\n" + "SELECT 1 AS order_id\n" + ), + "models/marts/migrated.sql": ( + "MODEL (materialized incremental, incremental_strategy append," + " columns (order_id (migrate_from id)));\nSELECT 1 AS order_id\n" + ), +} + + +HEADER_CONTRACT_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "enums/status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + "audits/generic/expression_is_true.sql": ( + 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE NOT (@expression)\n' + ), + "python/quality.py": ( + "from sqlbuild.audits import AuditCase, audit_factory\n\n\n" + "@audit_factory\n" + "def order_checks():\n" + " return [\n" + ' AuditCase(name="positive_total", definition="expression_is_true",\n' + ' arguments={"expression": "total > 0"}),\n' + " ]\n" + ), + "models/order_amounts.sql": ( + "MODEL (columns (customer_id (type INTEGER)));\n" + "SELECT 1 AS customer_id, 'books' AS category, 2 AS amount\n" + ), + "models/category_amounts.sql": ( + "MODEL (\n columns (customer_id (type INTEGER)),\n" + " dynamic_columns (category_amounts (pivot_column category, value_column amount," + ' aggregate max, type INTEGER, name_pattern "^[a-z]+$")),\n);\n' + 'SELECT * FROM __ref("order_amounts")\n' + "PIVOT(MAX(amount) FOR category IN (ANY ORDER BY category))\n" + ), + "models/order_statuses.sql": ( + "MODEL (\n contract enforced,\n" + " columns (status (type order_status, audits [not_null]), total (type INTEGER)),\n" + " audit_factories [order_checks],\n);\n" + 'SELECT @enum("order_status").PLACED AS status, 3 AS total\n' + ), +} + + +OPT_OUT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[settings]\nrequire_sql_analysis = true\n\n' + "[path_defaults.legacy]\nsql_analysis = false\n" + ), + "models/header_opt_out.sql": "MODEL (\n sql_analysis false\n);\nSELECT 1 AS order_id\n", + "models/legacy/path_opt_out.sql": "MODEL ();\nSELECT 1 AS order_id\n", + "models/unparsed.sql": "MODEL (sql_analysis false);\nSELECT FROM WHERE (\n", + "models/invalid_call.sql": ( + "MODEL ();\nSELECT * FROM __ref(order_name)\nCROSS JOIN __table_fn('customer_orders')\n" + ), +} + + +MODEL_SCHEMA_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "enums/status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + "schemas/orders.sql": ( + "SCHEMA (\n name order_shape,\n description 'Order shape.',\n" + " columns (order_id (type INTEGER, nullable false), status (type order_status)),\n);\n" + ), + "models/marts/_sqlbuild/schemas/local.sql": ( + "SCHEMA (\n name marts_shape,\n columns (order_id (type INTEGER)),\n);\n" + ), + "models/orders.sql": ( + "MODEL (\n contract enforced,\n model_schema order_shape,\n" + " columns (order_id (audits [unique]), note (type VARCHAR)),\n);\n" + "SELECT 1 AS order_id, 'PLACED' AS status, 'x' AS note\n" + ), + "models/staging/hidden.sql": ( + "MODEL (\n description 'Out of reach.',\n model_schema marts_shape,\n);\n" + "SELECT 1 AS order_id\n" + ), +} + + +PYTHON_HOOK_FILES: dict[str, str] = { + "hooks/python/notifications.py": ( + "from sqlbuild.hooks import hook\n\n\n" + "@hook\ndef notify(ctx, channel, *, level='info'):\n" + ' """Notify a channel."""\n return None\n\n\n' + "@hook\ndef positional(warehouse, /, ctx):\n" + ' """Take a positional-only warehouse."""\n return None\n\n\n' + "@hook\ndef flexible(ctx, **options):\n" + ' """Accept any configured argument."""\n return None\n' + ), +} + + +PYTHON_HOOK_PARITY_FILES: dict[str, str] = PYTHON_HOOK_FILES | { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/marts/_sqlbuild/hooks/python/local.py": ( + "from sqlbuild.hooks import hook\n\n\n" + "@hook\ndef refresh(ctx):\n" + ' """Refresh marts."""\n return None\n' + ), + "models/orders.sql": ( + 'MODEL (\n pre_hooks [python("notify", channel: "ops")],\n' + ' post_hooks [python("flexible", any: 1), python("notify", channel: "c", level: "x")],\n' + ");\n" + "SELECT 1 AS order_id\n" + ), + "models/marts/summary.sql": ( + 'MODEL (post_hooks [python("refresh"), python("flexible")]);\nSELECT 1 AS order_id\n' + ), + "models/staging/hidden.sql": ( + 'MODEL (post_hooks [python("refresh")]);\nSELECT 1 AS order_id\n' + ), +} + + +SQL_HOOK_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nregion = "eu"\n', + "enums/status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + "constants/limits.sql": "CONSTANT (name order_cap, value 3);\n", + "macros/money.py": 'def cents(value: str) -> str:\n return f"{value} * 100"\n', + "hooks/sql/grant_read.sql": ( + "HOOK (description 'Grant read access.');\n\n" + "GRANT SELECT ON @@CTX:destination.qualified TO @'role'\n" + ), + "hooks/sql/log_load.sql": ( + "HOOK (description 'Log a load.');\n\n" + 'INSERT INTO @target SELECT @enum("order_status").PLACED, @const("order_cap"),' + " @cents(@amount), '@@region' FROM __ref(\"stg_orders\")\n" + ), + "models/staging/stg_orders.sql": "MODEL ();\nSELECT 1 AS order_id\n", + "models/marts/loaded.sql": ( + "MODEL (\n schema analytics,\n" + ' pre_hooks [sql("grant_read", role: ["reader", "writer"])],\n' + ' post_hooks [sql("log_load", target: "audit.loads", amount: 5),' + ' inline_sql(\'SELECT @const("order_cap") FROM __ref("stg_orders")\')],\n);\n' + 'SELECT order_id FROM __ref("stg_orders")\n' + ), +} + +MACRO_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nregion = "eu"\n', + "macros/amounts.py": ( + '"""Amount macros."""\n\n\n' + 'def cents(value):\n """Cents."""\n return f"{value} * 100"\n\n\n' + 'def word():\n """A word."""\n return "x"\n' + ), +} + + +def build_inputs_with_native_loop( + *, project_dir: Path, monkeypatch: pytest.MonkeyPatch, defer_validation: bool = True +) -> tuple[CompileInputsView, list[int]]: + """Build compile inputs, counting the models the native loop finishes.""" + + finished: list[int] = [] + native_loop_type: type = _native.NativeModelLoop + + def counting_loop(*arguments: Any) -> _CountingLoop: + return _CountingLoop(native_loop_type(*arguments), finished) + + with monkeypatch.context() as patch: + patch.setattr(_native, "NativeModelLoop", counting_loop) + inputs: CompileInputsView = build_compile_inputs( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter_context=CompileAdapterContext( + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, + ), + defer_model_sql_validation=defer_validation, + no_cache=True, + ) + return inputs, finished + + +@dataclass(frozen=True) +class _CountingLoop: + native_loop: Any + finished: list[int] + + def run(self, *arguments: object) -> tuple[list[object], tuple[int, int, int]]: + inputs, counts = self.native_loop.run(*arguments) + self.finished.append(counts[0]) + return inputs, counts + + +_SUMMARY_FIELDS: tuple[str, ...] = ( + "config", + "query_sql", + "macro_source_sql", + "references", + "schema_entry", + "sql_validation_enabled", + "rejected_sql_analysis_opt_out", + "enum_declarations", + "constant_declarations", + "enum_columns", + "macro_deps", + "macro_usages", + "declaration_usages", + "sql_expansion", + "lint_expansion", +) + + +def inputs_summary(*, inputs: CompileInputsView, project_dir: Path) -> dict[str, object]: + """Each model input's fields and the diagnostics, as text with the project path elided.""" + + return { + "models": { + model_input.model_file.relative_path.as_posix(): _model_input_fields( + model_input=model_input, project_dir=project_dir + ) + for model_input in inputs.model_inputs + }, + "diagnostics": [ + _elided(value=item, project_dir=project_dir) for item in inputs.diagnostics + ], + } + + +def _model_input_fields(*, model_input: object, project_dir: Path) -> dict[str, str]: + return { + field_name: _elided(value=getattr(model_input, field_name), project_dir=project_dir) + for field_name in _SUMMARY_FIELDS + } + + +def _elided(*, value: object, project_dir: Path) -> str: + return repr(value).replace(str(project_dir), "") + + +def expected_summary(name: str) -> dict[str, object]: + """The recorded summary of the attachment's compile inputs for one parity project.""" + + return json.loads((EXPECTED_SUMMARIES / f"{name}.json").read_text(encoding="utf-8")) + + +EXPECTED_SUMMARIES: Path = Path(__file__).parent / "expected" + + +WAFFLE_SHOP_NATIVE_MODELS: int = 13 + + +def copy_project(*, fixtures: tuple[str, ...], files: dict[str, str], project_dir: Path) -> Path: + """Copy each fixture into `project_dir`, then write `files` over them.""" + + project_dir.mkdir(parents=True) + for fixture in fixtures: + _ = shutil.copytree(fixture, project_dir, dirs_exist_ok=True) + write_files(project_dir=project_dir, files=files) + return project_dir + + +SOURCE_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nregion = "eu"\n\n' + '[defaults]\ncontract = "enforced"\n' + ), + "enums/status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + "constants/limits.sql": "CONSTANT (name order_cap, value 3);\n", + "macros/money.py": 'def cents(value: str) -> str:\n return f"{value} * 100"\n', +} +_SOURCE_CONNECTION: dict[str, object] = {"database": "warehouse", "schema": "main"} + + +def compile_source_project( + *, project_dir: Path, source_yml: str, monkeypatch: pytest.MonkeyPatch +) -> tuple[CompiledProjectView, CompileInputsView]: + """Compile a project declaring `source_yml`; return the project and its compile inputs.""" + + write_files( + project_dir=project_dir, files={**SOURCE_PROJECT_FILES, "sources/raw.yml": source_yml} + ) + project: CompiledProjectView = compile_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection=_SOURCE_CONNECTION, + ) + inputs, _ = build_inputs_with_native_loop(project_dir=project_dir, monkeypatch=monkeypatch) + return project, inputs + + +def column_metadata(column: SourceColumnEntry) -> tuple[str | None, tuple[dict[str, object], ...]]: + """A source column's description and its audits' arguments.""" + + return column.description, tuple(audit.arguments for audit in column.audits) + + +FUNCTION_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nregion = "eu"\n' + 'amount_type = "DECIMAL(10, 2)"\n\n[defaults]\nfunction_schema = "udfs_${region}"\n' + ), + "enums/status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + "models/orders.sql": "MODEL (description 'Orders.', schema analytics);\nSELECT 1 AS order_id\n", + "functions/sql/order_total.sql": ( + "FUNCTION (description 'Order total.', tags [finance]," + " arguments (amount ${amount_type}), returns ${amount_type});\nSELECT amount * 2\n" + ), + "functions/sql/open_orders.sql": ( + "FUNCTION (description 'Open orders.', schema reporting," + " returns table (order_id INTEGER, status VARCHAR));\n" + 'SELECT order_id, @enum("order_status").PLACED AS status FROM __ref("orders")\n' + ), + "functions/python/score.py": ( + "from sqlbuild.functions import udf\n\n" + "@udf(arguments={'value': 'INTEGER'}, returns='INTEGER', runtime_version='3.11'," + " packages=['numpy'], tags=['ml'])\n" + "def main(value):\n '''Score.'''\n return value\n" + ), +} + + +def compile_function_project( + *, project_dir: Path, files: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> tuple[CompiledProjectView, CompileInputsView]: + """Compile a project of `files`; return the project and its compile inputs.""" + + write_files(project_dir=project_dir, files=files) + project: CompiledProjectView = compile_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection=_SOURCE_CONNECTION, + ) + inputs, _ = build_inputs_with_native_loop(project_dir=project_dir, monkeypatch=monkeypatch) + return project, inputs + + +def function_summary(function: CompiledFunctionView) -> tuple[object, ...]: + """A compiled function's planner-facing facts.""" + + return ( + type(function).__name__, + str(function.key.resource_type), + tuple(dep.name for dep in function.deps), + tuple((argument.name, argument.type) for argument in function.arguments), + function.returns, + tuple((column.name, column.type) for column in function.return_columns), + tuple(reference.ref_name for reference in function.references), + function.language.value, + function.destination.qualified_name, + function.fingerprint_destination.qualified_name, + (function.runtime_version, function.packages, function.tags, function.description), + ) + + +AUDIT_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[settings]\ndefault_audit_severity = "warn"\n' + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', columns (order_id (type INTEGER," + " audits [not_null, at_least(minimum 1, severity error)])));\nSELECT 1 AS order_id\n" + ), + "models/customers.sql": "MODEL (description 'Customers.');\nSELECT 1 AS customer_id\n", + "audits/generic/at_least.sql": ( + "AUDIT (name at_least);\nSELECT * FROM @relation WHERE @column < @minimum\n" + ), + "audits/singular/orders_have_customers.sql": ( + "AUDIT (name orders_have_customers, run_scope final);\n" + 'SELECT * FROM __ref("orders") o LEFT JOIN __ref("customers") c' + " ON o.order_id = c.customer_id WHERE c.customer_id IS NULL\n" + ), +} + + +def audit_summary(audit: CompiledAuditView) -> tuple[object, ...]: + """A compiled audit's planner-facing facts.""" + + return ( + type(audit).__name__, + audit.definition_name, + tuple(dep.name for dep in audit.scope_deps), + str(audit.severity), + audit.run_scope, + (audit.attached_target_name, audit.attached_column_name), + audit.sql_body, + ) + + +def measurement_policy(audit: CompiledAuditView) -> tuple[object, ...]: + """A measurement audit's bound operators and limits, sample count and severity.""" + + thresholds: MeasurementThresholds | None = audit.thresholds + assert thresholds is not None + return ( + *_bound_parts(thresholds.warn), + *_bound_parts(thresholds.error), + audit.minimum_samples, + str(audit.severity), + ) + + +def _bound_parts(bound: MeasurementThresholdBound | None) -> tuple[object, object]: + return (getattr(getattr(bound, "operator", None), "value", None), getattr(bound, "limit", None)) + + +RESOURCE_DIAGNOSTIC_CODES: frozenset[str] = frozenset({"P010", "P011", "C216"}) +RESOURCE_PROJECT_FILE: str = 'name = "orders"\nadapter = "duckdb"\n' + + +def resource_diagnostics_of( + *, project_dir: Path, files: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> tuple[tuple[object, ...], ...]: + """Each P010, P011 and C216 diagnostic of compiling `files`, as code, message and location.""" + + write_files( + project_dir=project_dir, files={"sqlbuild_project.toml": RESOURCE_PROJECT_FILE, **files} + ) + inputs, _ = build_inputs_with_native_loop(project_dir=project_dir, monkeypatch=monkeypatch) + resource_diagnostics: Iterator[CompilerDiagnostic] = filter( + lambda diagnostic: diagnostic.code in RESOURCE_DIAGNOSTIC_CODES, inputs.diagnostics + ) + return tuple( + ( + diagnostic.code, + diagnostic.message, + str(diagnostic.path), + diagnostic.line, + diagnostic.column, + getattr(diagnostic.location, "end_column", None), + ) + for diagnostic in resource_diagnostics + ) + + +_CURSOR_MODEL: str = ( + "MODEL (\n" + ' description "Daily revenue",\n' + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor revenue_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ");\n\n" + "SELECT CAST('2026-04-01' AS DATE) AS revenue_date{read}\n" +) + + +def cursor_model_files(*, read: str = "") -> dict[str, str]: + """An upstream payments model and an incremental cursor model reading `read`.""" + + return { + "models/payments.sql": 'MODEL (description "Payments");\nSELECT 1 AS id\n', + "models/finance/daily_revenue.sql": _CURSOR_MODEL.format(read=read), + } + + +_HAND_BUILT_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() +_HAND_BUILT_CONFIG: ProjectConfig = ProjectConfig(name="orders", adapter="duckdb") + + +def _hand_built_model(name: str, sql: str) -> CompileModelInput: + relative_path: Path = Path(f"models/{name}.sql") + return CompileModelInput( + model_file=DiscoveredSqlModelFile( + file_path=Path("/repo") / relative_path, + relative_path=relative_path, + contents=f"MODEL (description 'Test model.');\n\n{sql}\n", + header_values={}, + header_column_locations={}, + output_column_locations={}, + query_sql=sql, + ), + config=CompileModelConfig(values={}), + query_sql=sql, + references=extract_sql_references(sql=sql, syntax=_HAND_BUILT_SYNTAX), + sql_validation_enabled=True, + ) + + +def hand_built_analysed_project(models: dict[str, str]) -> CompiledProjectView: + """Assemble hand-built model inputs, which cross the explicit analysis boundary.""" + + return assemble_compiled_project( + inputs=CompileProjectInputs( + project_config=_HAND_BUILT_CONFIG, + local_config=LocalConfig(), + sql_lexical_syntax=_HAND_BUILT_SYNTAX, + discovered_inputs=DiscoveredProjectInputs( + project_config=_HAND_BUILT_CONFIG, local_config=LocalConfig() + ), + model_inputs=tuple(map(_hand_built_model, models, models.values())), + ), + inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb"), + ) + + +def native_analysed_project(*, project_dir: Path, models: dict[str, str]) -> CompiledProjectView: + """Compile the same models from files through the native owner.""" + + write_files( + project_dir=project_dir, + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + **{ + f"models/{name}.sql": f"MODEL (description 'Test model.');\n\n{sql}\n" + for name, sql in models.items() + }, + }, + ) + return compile_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + resolved_connection=_SOURCE_CONNECTION, + ) + + +def model_analysis_summary(project: CompiledProjectView) -> tuple[tuple[object, ...], ...]: + """Each model's column names, lineage outputs, star flag and located binding diagnostics.""" + + return tuple(map(_model_analysis, sorted(project.models, key=_model_name))) + + +def _model_name(model: CompiledModelView) -> str: + return model.name + + +def _model_analysis(model: CompiledModelView) -> tuple[object, ...]: + return ( + model.name, + tuple(column.name for column in model.inferred_columns or ()), + tuple(column.output_column for column in model.fast_lineage_columns or ()), + model.fast_lineage_has_star, + model.binding_validated, + tuple(map(_located_binding, model.binding_diagnostics)), + ) + + +def _located_binding(diagnostic: CompilerDiagnostic) -> tuple[object, ...]: + return ( + diagnostic.code, + diagnostic.line, + diagnostic.column, + getattr(diagnostic.location, "end_column", None), + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py new file mode 100644 index 0000000000..212d0679cf --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_audits.py @@ -0,0 +1,270 @@ +"""Audits attached by the native model loop and assembled into the native compiled project.""" + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.types import AuditInput, CompiledAuditView +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + NativeAuditAttachmentTestCase, + NativeAuditErrorTestCase, + NativeMeasurementPolicyTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + AUDIT_PROJECT_FILES, + audit_summary, + compile_function_project, + measurement_policy, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeAuditAttachmentTestCase( + description="singular audit reading two models under its own run scope", + audit_name="orders_have_customers", + expected_summary=( + "NativeCompiledAudit", + "orders_have_customers", + ("orders", "customers"), + "warn", + "final", + (None, None), + 'SELECT * FROM __ref("orders") o LEFT JOIN __ref("customers") c' + " ON o.order_id = c.customer_id WHERE c.customer_id IS NULL", + ), + expected_usages=( + "singular_audit:orders_have_customers:customers", + "singular_audit:orders_have_customers:orders", + ), + ), + NativeAuditAttachmentTestCase( + description="built-in column audit under the project default severity", + audit_name="not_null", + expected_summary=( + "NativeCompiledAudit", + "not_null", + ("orders",), + "warn", + "delta_and_final", + ("orders", "order_id"), + 'SELECT order_id\nFROM __ref("orders")\nWHERE order_id IS NULL', + ), + expected_usages=(), + ), + NativeAuditAttachmentTestCase( + description="project generic audit with an argument and its own severity", + audit_name="at_least", + expected_summary=( + "NativeCompiledAudit", + "at_least", + ("orders",), + "error", + "delta_and_final", + ("orders", "order_id"), + 'SELECT * FROM __ref("orders") WHERE order_id < 1', + ), + expected_usages=("audit:at_least:orders",), + ), + ], + ids=lambda case: case.description, +) +def test_given_audit_when_compiling_then_native_project_holds_its_attachment( + test_case: NativeAuditAttachmentTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project, inputs = compile_function_project( + project_dir=tmp_path, files=AUDIT_PROJECT_FILES, monkeypatch=monkeypatch + ) + + audits: dict[str, CompiledAuditView] = {audit.name: audit for audit in project.audits} + audit_inputs: dict[str, AuditInput] = { + audit.definition_name: audit_input + for audit, audit_input in zip(project.audits, inputs.audit_inputs, strict=True) + } + assert audit_summary(audits[test_case.audit_name]) == test_case.expected_summary + assert ( + tuple( + f"{usage.declaration.kind.value}:{usage.declaration.name}:{usage.consumer.name}" + for usage in audit_inputs[test_case.audit_name].declaration_usages + ) + == test_case.expected_usages + ) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeAuditErrorTestCase( + description="attachment of an unknown generic audit", + files={ + "models/customers.sql": ( + "MODEL (description 'Customers.', columns (customer_id (type INTEGER," + " audits [at_most(maximum 3)])));\nSELECT 1 AS customer_id\n" + ), + }, + expected_message="references unknown generic audit 'at_most'", + ), + NativeAuditErrorTestCase( + description="singular audit reading an unknown model", + files={ + "audits/singular/orders_have_customers.sql": ( + "AUDIT (name orders_have_customers);\n" + 'SELECT * FROM __ref("orders") JOIN __ref("missing_customers") ON TRUE\n' + ), + }, + expected_message=( + "Audit file audits/singular/orders_have_customers.sql references unknown model" + " 'missing_customers'" + ), + ), + NativeAuditErrorTestCase( + description="unknown project default run scope", + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[settings]\n' + 'default_audit_run_scope = "always"\n' + ), + }, + expected_message=( + "settings.default_audit_run_scope in sqlbuild_project.toml: unknown value 'always'" + ), + ), + NativeAuditErrorTestCase( + description="unknown singular audit run scope", + files={ + "audits/singular/orders_have_customers.sql": ( + "AUDIT (name orders_have_customers, run_scope always);\n" + 'SELECT * FROM __ref("orders") JOIN __ref("customers") ON TRUE\n' + ), + }, + expected_message="unknown audit run_scope 'always'; valid values: delta_and_final, final", + ), + NativeAuditErrorTestCase( + description="model-attached audit that does not read its model", + files={ + "audits/generic/at_least.sql": ( + 'AUDIT (name at_least);\nSELECT * FROM __ref("customers") WHERE @minimum > 0\n' + ), + }, + expected_message=( + "models/orders.sql audit 'at_least': model-attached audit must reference the" + " attached model 'orders' via __ref()" + ), + ), + NativeAuditErrorTestCase( + description="singular audit reading a dbt model", + files={ + "audits/singular/orders_have_customers.sql": ( + "AUDIT (name orders_have_customers);\n" + 'SELECT * FROM __ref("orders") JOIN __dbt_ref("customers") ON TRUE\n' + ), + }, + expected_message=( + 'may not use __dbt_ref("customers"); audit dbt model checks belong in dbt' + ), + ), + NativeAuditErrorTestCase( + description="thresholds on a violation audit attachment", + files={ + "models/orders.sql": ( + "MODEL (description 'Orders.', columns (order_id (type INTEGER," + " audits [at_least(minimum 1, thresholds (warn (above 1)))])));\n" + "SELECT 1 AS order_id\n" + ), + }, + expected_message="thresholds and minimum_samples are only valid for measurement audits", + ), + NativeAuditErrorTestCase( + description="standalone measurement audit with a non-numeric threshold", + files={ + "audits/singular/coverage.sql": 'AUDIT (name coverage, evaluation measurement, value rate, thresholds (warn (above \'x\')));\nMEASURE (SELECT COUNT(*) AS rate FROM __ref("orders") JOIN __ref("customers") ON TRUE);\n' + }, + expected_message=( + "audits/singular/coverage.sql standalone measurement audit invalid thresholds:" + " above threshold requires one numeric value" + ), + ), + NativeAuditErrorTestCase( + description="standalone measurement audit with negative minimum samples", + files={ + "audits/singular/coverage.sql": 'AUDIT (name coverage, evaluation measurement, value rate, thresholds (warn (above 1)), minimum_samples -1);\nMEASURE (SELECT COUNT(*) AS rate FROM __ref("orders") JOIN __ref("customers") ON TRUE);\n' + }, + expected_message=( + "audits/singular/coverage.sql standalone measurement audit 'minimum_samples'" + " must be a non-negative integer" + ), + ), + NativeAuditErrorTestCase( + description="generic audit attachment missing an argument", + files={ + "models/orders.sql": ( + "MODEL (description 'Orders.', columns (order_id (type INTEGER, audits [at_least])));\n" + "SELECT 1 AS order_id\n" + ) + }, + expected_message="models/orders.sql is missing argument 'minimum' for generic audit 'at_least'", + ), + NativeAuditErrorTestCase( + description="generic audit attachment overriding its implicit column", + files={ + "models/orders.sql": ( + "MODEL (description 'Orders.', columns (order_id (type INTEGER, audits [at_least(minimum 1, column other_id)])));\n" + "SELECT 1 AS order_id\n" + ) + }, + expected_message=( + "models/orders.sql audit 'at_least' must not override implicit column from" + " attached context" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_audit_when_compiling_then_native_attachment_rejects_it( + test_case: NativeAuditErrorTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with pytest.raises(CompileInputError) as raised: + _ = compile_function_project( + project_dir=tmp_path, + files={**AUDIT_PROJECT_FILES, **test_case.files}, + monkeypatch=monkeypatch, + ) + + assert test_case.expected_message in str(raised.value) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeMeasurementPolicyTestCase( + description="error threshold makes the standalone audit an error", + files={ + "audits/singular/coverage.sql": 'AUDIT (name coverage, evaluation measurement, value rate, thresholds (warn (above 1), error (above 5)), minimum_samples 3);\nMEASURE (SELECT COUNT(*) AS rate FROM __ref("orders") JOIN __ref("customers") ON TRUE);\n' + }, + expected_policy=("above", 1.0, "above", 5.0, 3, "error"), + ), + NativeMeasurementPolicyTestCase( + description="warn-only threshold keeps an unbounded authored sample count", + files={ + "audits/singular/coverage.sql": 'AUDIT (name coverage, evaluation measurement, value rate, thresholds (warn (above 1)), minimum_samples 99999999999999999999);\nMEASURE (SELECT COUNT(*) AS rate FROM __ref("orders") JOIN __ref("customers") ON TRUE);\n' + }, + expected_policy=("above", 1.0, None, None, 99999999999999999999, "warn"), + ), + ], + ids=lambda case: case.description, +) +def test_given_standalone_measurement_audit_when_compiling_then_header_policy_is_parsed( + test_case: NativeMeasurementPolicyTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project, _ = compile_function_project( + project_dir=tmp_path, + files={**AUDIT_PROJECT_FILES, **test_case.files}, + monkeypatch=monkeypatch, + ) + + audit: CompiledAuditView = {audit.name: audit for audit in project.audits}["coverage"] + assert measurement_policy(audit) == test_case.expected_policy diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py new file mode 100644 index 0000000000..be91022484 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -0,0 +1,615 @@ +"""The rules request reads model facts the compile-input stage retained natively.""" + +import json +import pickle +from collections import Counter +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.cli.entry.main.entry import main +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.models import InferredColumn +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.pipeline.main.project import compile_project +from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + ImmediateSyntaxErrorTestCase, + ModelLoopErrorTestCase, + ModelLoopParityTestCase, + ModelPathScopeErrorTestCase, + RetainedRulesRowsTestCase, + SharedInferredColumnsTestCase, + UnretainedRulesRowsTestCase, + ZeroMaterializationTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + HEADER_CONTRACT_FILES, + LEGACY_PROJECT_TYPES, + MACRO_FILES, + MODEL_SCHEMA_FILES, + OPT_OUT_FILES, + PARITY_FILES, + PYTHON_HOOK_FILES, + PYTHON_HOOK_PARITY_FILES, + SQL_HOOK_FILES, + TYPE_PROOF_FILES, + WAFFLE_SHOP_NATIVE_MODELS, + build_inputs_with_native_loop, + copy_project, + count_legacy_constructions, + expected_summary, + forbid_hand_built_projects, + inputs_summary, + skip_model_retention, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import ORDERS_PROJECT_FILES, write_files + + +@pytest.mark.parametrize( + "test_case", + [SharedInferredColumnsTestCase("repeated output schemas share pickle entries", 24, 3)], + ids=lambda case: case.description, +) +def test_given_repeated_output_schemas_when_serializing_then_columns_share_immutable_values( + test_case: SharedInferredColumnsTestCase, + tmp_path: Path, +) -> None: + write_files( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + **{ + f"models/orders_{index}.sql": ( + 'MODEL (description "Order identifiers");\n' + "SELECT CAST(1 AS INT) AS order_id, CAST(NULL AS INT) AS missing_id, " + "CAST('ready' AS VARCHAR) AS status" + ) + for index in range(test_case.model_count) + }, + }, + ) + project: CompiledProjectView = compile_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + schemas: tuple[tuple[InferredColumn, ...] | None, ...] = tuple( + model.inferred_columns for model in project.models + ) + restored: tuple[tuple[InferredColumn, ...] | None, ...] = pickle.loads(pickle.dumps(schemas)) + assert restored == schemas + columns: list[InferredColumn] = [] + for schema in restored: + columns.extend(schema or ()) + assert len(columns) == test_case.model_count * test_case.expected_columns_per_model + assert len(set(columns)) == test_case.expected_columns_per_model + assert len({id(column) for column in columns}) == test_case.expected_columns_per_model + + +@pytest.mark.parametrize( + "test_case", + [ + RetainedRulesRowsTestCase( + description="compile proves the passthrough from retained facts", + expected_codes=("K002", "SQBRCONTRACT105"), + ), + ], + ids=lambda case: case.description, +) +def test_given_compiled_project_when_evaluating_rules_then_retained_model_facts_are_read( + test_case: RetainedRulesRowsTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + forbid_hand_built_projects(monkeypatch=monkeypatch) + + _ = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes + + +@pytest.mark.parametrize( + "test_case", + [ + UnretainedRulesRowsTestCase( + description="models the compile-input stage did not retain", + expected_error="compiled model 'orders' has no retained compile facts", + ) + ], + ids=lambda case: case.description, +) +def test_given_unretained_model_facts_when_evaluating_rules_then_compile_reports_it( + test_case: UnretainedRulesRowsTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + skip_model_retention(monkeypatch=monkeypatch) + + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) + output: str = "".join(capsys.readouterr()) + + assert exit_code == 1 + assert test_case.expected_error in output + + +@pytest.mark.parametrize( + "test_case", + [ + ModelLoopParityTestCase( + description="plain, interpolated, seeded, incremental, column-contract, enum, constant, macro and hooked models", + fixtures=(), + files=PARITY_FILES, + expected_native_models=9, + expected_summary="parity", + ), + ModelLoopParityTestCase( + description="dynamic columns, audit factories and enforced enum columns", + fixtures=(), + files=HEADER_CONTRACT_FILES, + expected_native_models=3, + expected_summary="header_contract", + ), + ModelLoopParityTestCase( + description="accepted and rejected analysis opt-outs and invalid reference calls", + fixtures=(), + files=OPT_OUT_FILES, + expected_native_models=4, + expected_summary="opt_out", + ), + ModelLoopParityTestCase( + description="named model schemas, augmented and out of scope", + fixtures=(), + files=MODEL_SCHEMA_FILES, + expected_native_models=2, + expected_summary="model_schema", + ), + ModelLoopParityTestCase( + description="python hooks, checked and scoped", + fixtures=(), + files=PYTHON_HOOK_PARITY_FILES, + expected_native_models=3, + expected_summary="python_hooks", + ), + ModelLoopParityTestCase( + description="named and inline SQL hooks with arguments, context, declarations and macros", + fixtures=(), + files=SQL_HOOK_FILES, + expected_native_models=2, + expected_summary="sql_hooks", + ), + ModelLoopParityTestCase( + description="neutral fixture with macros, enums and declared columns", + fixtures=("tests/e2e/fixtures/waffle_shop",), + files={}, + expected_native_models=WAFFLE_SHOP_NATIVE_MODELS, + expected_summary="waffle_shop", + ), + ], + ids=lambda case: case.description, +) +def test_given_project_when_running_native_model_loop_then_inputs_match_recorded_attachment( + test_case: ModelLoopParityTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_dir: Path = copy_project( + fixtures=test_case.fixtures, files=test_case.files, project_dir=tmp_path / "project" + ) + + native, native_counts = build_inputs_with_native_loop( + project_dir=project_dir, monkeypatch=monkeypatch + ) + + assert native_counts == [test_case.expected_native_models] + assert inputs_summary(inputs=native, project_dir=project_dir) == expected_summary( + test_case.expected_summary + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ModelLoopErrorTestCase( + description="blank header description", + model_sql="MODEL (description ' ');\nSELECT 1 AS order_id\n", + expected_message="models/orders.sql model 'description' must be a non-empty string", + expected_code="P001", + ), + ModelLoopErrorTestCase( + description="conflicting analysis overrides", + model_sql="MODEL (sql_analysis true, sql_validation false);\nSELECT 1 AS order_id\n", + expected_message="MODEL sql_analysis conflicts with legacy sql_validation", + expected_code="P003", + ), + ModelLoopErrorTestCase( + description="column rename on a rebuilt table", + model_sql=( + "MODEL (materialized table, columns (revenue (migrate_from amount)));\n" + "SELECT 1 AS revenue\n" + ), + expected_message=( + "model 'orders': column 'revenue' declares migrate_from, which is only valid for " + "incremental and snapshot models, or for table and view models that declare " + "migrate_from themselves (the old name's compatibility view then presents the " + "column under its old name); 'table' models are rebuilt with their new columns" + ), + expected_code="P001", + ), + ModelLoopErrorTestCase( + description="invalid dynamic column pattern", + model_sql=( + "MODEL (dynamic_columns (amounts (pivot_column category, value_column amount," + ' aggregate max, type INTEGER, name_pattern "(")));\n' + "SELECT 1 AS order_id\n" + ), + expected_message=( + "model 'orders' dynamic column family 'amounts' has invalid name_pattern: " + "unclosed group" + ), + expected_code="P001", + ), + ModelLoopErrorTestCase( + description="unknown audit factory", + model_sql="MODEL (audit_factories [order_checks]);\nSELECT 1 AS order_id\n", + expected_message=( + "model 'orders' in models/orders.sql references unknown audit factory " + "'order_checks'. Available audit factories: none discovered" + ), + expected_code="P001", + ), + ModelLoopErrorTestCase( + description="unknown model schema", + model_sql="MODEL (model_schema order_shape);\nSELECT 1 AS order_id\n", + expected_message=( + "model 'orders' references unknown model_schema 'order_shape'; " + "available schemas: none" + ), + expected_code="P001", + ), + ModelLoopErrorTestCase( + description="dbt reference without a dbt manifest", + model_sql='MODEL ();\nSELECT * FROM __dbt_ref("raw_orders")\n', + expected_message=( + 'Model file models/orders.sql uses __dbt_ref("raw_orders") but no dbt manifest ' + "was found" + ), + expected_code="C214", + ), + ModelLoopErrorTestCase( + description="unclosed reference call located in the model file", + model_sql='MODEL ();\nSELECT *\nFROM __ref("stg_orders"\n', + expected_message="models/orders.sql:3:6: SQL reference contains an unclosed parenthesis", + expected_code="P001", + ), + ModelLoopErrorTestCase( + description="unknown project variable", + model_sql="MODEL ();\nSELECT '@@missing_var' AS label\n", + expected_message=( + "unknown project variable '@@missing_var' in '{model_path}'. Available vars: none" + ), + expected_code="P001", + ), + ModelLoopErrorTestCase( + description="python hook that is not discovered", + model_sql='MODEL (post_hooks [python("flexible"), python("missing")]);\nSELECT 1 AS id\n', + expected_message=( + "model 'orders' post_hooks[1] python(\"missing\") references an unknown hook. " + "Discovered hooks: flexible, notify, positional" + ), + expected_code="P001", + files=PYTHON_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="python hook argument shadowing a context parameter", + model_sql=( + 'MODEL (pre_hooks [python("notify", hook_context: "x", ctx: "y")]);\n' + "SELECT 1 AS id\n" + ), + expected_message=( + "model 'orders' pre_hooks[0] python(\"notify\") argument 'ctx' conflicts with " + "reserved context parameter 'ctx'. Rename the hook argument; context parameters " + "are injected by SQLBuild." + ), + expected_code="P001", + files=PYTHON_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="python hook arguments the function does not accept", + model_sql=( + 'MODEL (pre_hooks [python("notify", zone: "x", channel: "c", area: "y")]);\n' + "SELECT 1 AS id\n" + ), + expected_message=( + "model 'orders' pre_hooks[0] python(\"notify\") has unknown argument(s): area, " + "zone. Accepted configured arguments: channel, level" + ), + expected_code="P001", + files=PYTHON_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="python hook with a required positional-only parameter", + model_sql='MODEL (post_hooks [python("positional")]);\nSELECT 1 AS id\n', + expected_message=( + "model 'orders' post_hooks[0] python(\"positional\") cannot be configured " + "because the hook function has required positional-only parameter(s): " + "warehouse. Use keyword-capable parameters for values supplied from MODEL hooks." + ), + expected_code="P001", + files=PYTHON_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="python hook missing a required argument after valid hooks", + model_sql=( + 'MODEL (post_hooks [python("flexible", any: 1)], pre_hooks [python("notify")]);\n' + "SELECT 1 AS id\n" + ), + expected_message=( + "model 'orders' pre_hooks[0] python(\"notify\") is missing required " + "argument(s): channel" + ), + expected_code="P001", + files=PYTHON_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="macro arguments located in the authored model", + model_sql="MODEL ();\nSELECT @cents(1 +) AS amount\n", + expected_message=( + "Macro arguments of '@cents' in '{model_path}' must use only Python literals, " + "nested macro calls, and __ref(), __source(), or __seed() references at line 2, " + "column 17" + ), + expected_code="P001", + files=MACRO_FILES, + ), + ModelLoopErrorTestCase( + description="unary sign over a nested macro returning text", + model_sql="MODEL ();\nSELECT @cents(-@word()) AS amount\n", + expected_message=( + "Macro arguments of '@cents' in '{model_path}' apply unary - to the value of a " + "nested macro call, which is a str and not a number at line 2, column 16" + ), + expected_code="P001", + files=MACRO_FILES, + ), + ModelLoopErrorTestCase( + description="macro arguments in interpolated SQL stay relative to the call", + model_sql="MODEL ();\nSELECT '@@region' AS region, @cents(1 +) AS amount\n", + expected_message=( + "Macro arguments in '{model_path}' must use only Python literals, nested macro " + "calls, and __ref(), __source(), or __seed() references at line 1, column 3 of " + "the '@cents' arguments" + ), + expected_code="P001", + files=MACRO_FILES, + ), + ModelLoopErrorTestCase( + description="named SQL hook missing an argument", + model_sql='MODEL (post_hooks [sql("grant_read")]);\nSELECT 1 AS id\n', + expected_message=( + "{model_path} post_hooks[0] sql(\"grant_read\") is missing argument 'role' for " + "SQL hook 'grant_read' in hooks/sql/grant_read.sql" + ), + expected_code="P001", + files=SQL_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="named SQL hook passed an argument it does not use", + model_sql=( + 'MODEL (pre_hooks [sql("grant_read", role: "reader", scope: "all", mode: 1)]);\n' + "SELECT 1 AS id\n" + ), + expected_message=( + '{model_path} pre_hooks[0] sql("grant_read") passes unknown argument(s) to ' + "SQL hook 'grant_read' in hooks/sql/grant_read.sql: mode, scope" + ), + expected_code="P001", + files=SQL_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="unknown named SQL hook", + model_sql='MODEL (post_hooks [sql("vacuum")]);\nSELECT 1 AS id\n', + expected_message=( + "post_hooks[0] sql(\"vacuum\") in '{model_path}' references an unknown SQL hook. " + "Discovered SQL hooks: grant_read, log_load. If this is an inline SQL statement, " + 'use inline_sql("...").' + ), + expected_code="P001", + files=SQL_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="inline SQL hook with a config template", + model_sql="MODEL (post_hooks [inline_sql('SELECT ${region}')]);\nSELECT 1 AS id\n", + expected_message=( + "post_hooks[0] inline_sql(\"...\") in '{model_path}' uses unsupported ${{...}} " + "template syntax. Use @@CTX:..., @@ENV:..., or @@project_var inside SQL hooks." + ), + expected_code="P001", + files=SQL_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="inline SQL hook with an unknown project variable", + model_sql="MODEL (pre_hooks [inline_sql('SELECT @@missing')]);\nSELECT 1 AS id\n", + expected_message=( + "unknown project variable '@@missing' in '{model_path}'. Available vars: region" + ), + expected_code="P001", + files=SQL_HOOK_FILES, + ), + ModelLoopErrorTestCase( + description="unknown enum member", + model_sql='MODEL ();\nSELECT @enum("order_status").LOST AS status\n', + expected_message=( + "Unknown member 'LOST' for enum 'order_status' in '{model_path}'. " + "Available members: PLACED, SHIPPED" + ), + expected_code="P001", + ), + ModelLoopErrorTestCase( + description="unknown constant", + model_sql='MODEL ();\nSELECT @const("max_items") AS limit_value\n', + expected_message=( + "Unknown constant 'max_items' in '{model_path}'. Visible constants: none" + ), + expected_code="P001", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_model_when_running_native_model_loop_then_attachment_error_is_raised( + test_case: ModelLoopErrorTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_dir: Path = copy_project( + fixtures=(), + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "enums/status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + "models/orders.sql": test_case.model_sql, + } + | test_case.files, + project_dir=tmp_path / "project", + ) + + with pytest.raises(CompileInputError) as raised: + _ = build_inputs_with_native_loop(project_dir=project_dir, monkeypatch=monkeypatch) + + assert raised.value.message == test_case.expected_message.format( + model_path=project_dir / "models" / "orders.sql" + ) + assert raised.value.code == test_case.expected_code + + +@pytest.mark.parametrize( + "test_case", + [ + ImmediateSyntaxErrorTestCase( + description="query the parser rejects", + model_sql="MODEL ();\nSELECT FROM WHERE (\n", + expected_message=( + "SQL syntax error in model 'orders' ({model_path}): " + "Expected table name or subquery, got Where" + ), + expected_help_fragment="skip SQL analysis for this model", + ), + ImmediateSyntaxErrorTestCase( + description="incremental query with cursor intrinsics the parser rejects", + model_sql=( + "MODEL (materialized incremental, incremental_strategy append, cursor order_id," + " cursor_type integer);\n" + "SELECT 1 AS order_id FROM t WHERE order_id >= __cursor_start() AND (\n" + ), + expected_message=( + "SQL syntax error in model 'orders' ({model_path}): Unexpected token: LParen" + ), + expected_help_fragment="skip SQL analysis for this model", + ), + ], + ids=lambda case: case.description, +) +def test_given_unparseable_model_when_validating_immediately_then_syntax_error_is_raised( + test_case: ImmediateSyntaxErrorTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_dir: Path = copy_project( + fixtures=(), + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/orders.sql": test_case.model_sql, + }, + project_dir=tmp_path / "project", + ) + + with pytest.raises(CompileInputError) as raised: + _ = build_inputs_with_native_loop( + project_dir=project_dir, monkeypatch=monkeypatch, defer_validation=False + ) + + assert raised.value.message == test_case.expected_message.format( + model_path=project_dir / "models" / "orders.sql" + ) + assert test_case.expected_help_fragment in (raised.value.help or "") + + +@pytest.mark.parametrize( + "test_case", + [ + ZeroMaterializationTestCase( + description="ordinary compile", + arguments=("compile", "--no-cache"), + expected_constructions=dict.fromkeys(LEGACY_PROJECT_TYPES, 0), + ), + ZeroMaterializationTestCase( + description="selected compile with a JSON report", + arguments=("compile", "--no-cache", "--json", "--select", "orders+"), + expected_constructions=dict.fromkeys(LEGACY_PROJECT_TYPES, 0), + ), + ], + ids=lambda case: case.description, +) +def test_given_ordinary_compile_when_compiling_then_no_python_project_is_constructed( + test_case: ZeroMaterializationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_files(project_dir=tmp_path, files=ORDERS_PROJECT_FILES) + counts: Counter[str] = count_legacy_constructions(monkeypatch=monkeypatch) + + exit_code: int = main(["--project-dir", str(tmp_path), *test_case.arguments]) + _ = capsys.readouterr() + + assert exit_code == 0 + assert {name: counts[name] for name in LEGACY_PROJECT_TYPES} == ( + test_case.expected_constructions + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ModelPathScopeErrorTestCase( + description="a folder holding the identity separator is read as an identity", + model_path="models/a:b/orders.sql", + expected_message="Unknown qualified identity kind: 'models/a'", + expected_code="S002", + ), + ], + ids=lambda case: case.description, +) +def test_given_model_path_scope_rejects_when_running_native_model_loop_then_scope_error_is_raised( + test_case: ModelPathScopeErrorTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_dir: Path = copy_project( + fixtures=(), + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "enums/status.sql": "ENUM (name order_status, members [PLACED, SHIPPED]);\n", + test_case.model_path: "MODEL (description 'Orders.');\nSELECT 1 AS order_id\n", + }, + project_dir=tmp_path / "project", + ) + + with pytest.raises(InvalidQualifiedIdentityError) as raised: + _ = build_inputs_with_native_loop(project_dir=project_dir, monkeypatch=monkeypatch) + + assert (str(raised.value), raised.value.code) == ( + test_case.expected_message, + test_case.expected_code, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py new file mode 100644 index 0000000000..8ed82f5416 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_functions.py @@ -0,0 +1,150 @@ +"""Functions attached by the native model loop and assembled into the native compiled project.""" + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.types import CompiledFunctionView, FunctionInput +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + NativeFunctionAttachmentTestCase, + NativeFunctionErrorTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + FUNCTION_PROJECT_FILES, + compile_function_project, + function_summary, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeFunctionAttachmentTestCase( + description="scalar SQL function with templated types under the default schema", + function_name="order_total", + expected_summary=( + "NativeCompiledFunction", + "udf", + (), + (("amount", "DECIMAL(10, 2)"),), + "DECIMAL(10, 2)", + (), + (), + "sql", + "udfs_eu.order_total", + "udfs_eu.order_total", + (None, (), ("finance",), "Order total."), + ), + expected_usages=(), + ), + NativeFunctionAttachmentTestCase( + description="table SQL function reading a model and an enum in its own schema", + function_name="open_orders", + expected_summary=( + "NativeCompiledFunction", + "table_fn", + ("orders",), + (), + "TABLE", + (("order_id", "INTEGER"), ("status", "VARCHAR")), + ("orders",), + "sql", + "reporting.open_orders", + "reporting.open_orders", + (None, (), (), "Open orders."), + ), + expected_usages=("enum:order_status",), + ), + NativeFunctionAttachmentTestCase( + description="Python function fingerprinted in the default schema it does not inherit", + function_name="score", + expected_summary=( + "NativeCompiledFunction", + "udf", + (), + (("value", "INTEGER"),), + "INTEGER", + (), + (), + "python", + "score", + "udfs_eu.score", + ("3.11", ("numpy",), ("ml",), "Score."), + ), + expected_usages=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_function_when_compiling_then_native_project_holds_its_attachment( + test_case: NativeFunctionAttachmentTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project, inputs = compile_function_project( + project_dir=tmp_path, files=FUNCTION_PROJECT_FILES, monkeypatch=monkeypatch + ) + + function: CompiledFunctionView = {item.name: item for item in project.functions}[ + test_case.function_name + ] + function_input: FunctionInput = {item.name: item for item in inputs.sql_function_inputs}[ + test_case.function_name + ] + assert function_summary(function) == test_case.expected_summary + assert type(function_input).__name__ == "NativeFunctionInput" + assert ( + tuple( + f"{usage.declaration.kind.value}:{usage.declaration.name}" + for usage in function_input.declaration_usages + ) + == test_case.expected_usages + ) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeFunctionErrorTestCase( + description="table function called with the wrong argument count", + files={ + "functions/sql/customer_orders.sql": ( + "FUNCTION (description 'Customer orders.', arguments (customer_id INTEGER)," + " returns table (order_id INTEGER));\nSELECT customer_id AS order_id\n" + ), + "models/report.sql": ( + "MODEL (description 'Report.');\n" + 'SELECT * FROM __table_fn("customer_orders")(1, 2)\n' + ), + }, + expected_message=( + "Model file models/report.sql table function 'customer_orders' expects 1" + " argument but received 2" + ), + ), + NativeFunctionErrorTestCase( + description="function body reading an unknown model", + files={ + "functions/sql/lookup.sql": ( + "FUNCTION (description 'Lookup.', returns INTEGER);\n" + 'SELECT COUNT(*) FROM __ref("missing_orders")\n' + ), + }, + expected_message=( + "SQL function file functions/sql/lookup.sql references unknown model" + " 'missing_orders'" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_function_when_compiling_then_native_attachment_rejects_it( + test_case: NativeFunctionErrorTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with pytest.raises(CompileInputError) as raised: + _ = compile_function_project( + project_dir=tmp_path, + files={**FUNCTION_PROJECT_FILES, **test_case.files}, + monkeypatch=monkeypatch, + ) + + assert test_case.expected_message in str(raised.value) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_model_analysis_handoff.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_model_analysis_handoff.py new file mode 100644 index 0000000000..39dbdd8999 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_model_analysis_handoff.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + HandBuiltAnalysisTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + hand_built_analysed_project, + model_analysis_summary, + native_analysed_project, +) + +_ORDERS: str = "SELECT 1 AS order_id, 2 AS amount" + + +@pytest.mark.parametrize( + "test_case", + [ + HandBuiltAnalysisTestCase( + description="a typo locates its binding error in the authored file", + models={ + "orders": _ORDERS, + "report": 'SELECT o.order_id, o.missing\nFROM __ref("orders") AS o', + }, + expected_models=( + ("orders", ("order_id", "amount"), ("order_id", "amount"), False, True, ()), + ( + "report", + ("order_id", "missing"), + ("order_id", "missing"), + False, + True, + (("B002", 3, 22, 29),), + ), + ), + ), + HandBuiltAnalysisTestCase( + description="a clean star model keeps native columns and lineage", + models={"orders": _ORDERS, "report": 'SELECT *\nFROM __ref("orders")'}, + expected_models=( + ("orders", ("order_id", "amount"), ("order_id", "amount"), False, True, ()), + ("report", ("order_id", "amount"), ("order_id", "amount"), True, True, ()), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_models_when_assembling_natively_and_hand_built_then_analysis_facts_match( + test_case: HandBuiltAnalysisTestCase, tmp_path: Path +) -> None: + native: tuple[tuple[object, ...], ...] = model_analysis_summary( + native_analysed_project(project_dir=tmp_path, models=test_case.models) + ) + hand_built: tuple[tuple[object, ...], ...] = model_analysis_summary( + hand_built_analysed_project(test_case.models) + ) + + assert (native, hand_built) == (test_case.expected_models, test_case.expected_models) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py new file mode 100644 index 0000000000..fb7b8a8336 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_parameter_boundaries.py @@ -0,0 +1,123 @@ +"""Adapter literals and quoted hook arguments cross real native attachment boundaries.""" + +import json +from pathlib import Path +from typing import cast + +import pytest + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs +from sqlbuild.compiler.compile.models import CompileAdapterContext +from sqlbuild.compiler.compile.types import CompileInputsView +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import SqlHookEntry +from sqlbuild.sql_values.types import CollectionRendering +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + AdapterParameterBoundaryTestCase, + QuotedHookBoundaryTestCase, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files + + +@pytest.mark.parametrize( + "test_case", + [ + AdapterParameterBoundaryTestCase( + "decimal scale in BigQuery", + "bigquery", + "decimal", + '"12.3400"', + "SELECT NUMERIC '12.3400' AS value", + ), + AdapterParameterBoundaryTestCase( + "Unicode literal in SQL Server", + "sqlserver", + "string", + '"O\'Brien"', + "SELECT N'O''Brien' AS value", + ), + AdapterParameterBoundaryTestCase( + "boolean scalar in SQL Server", "sqlserver", "boolean", "true", "SELECT 1 AS value" + ), + ], + ids=lambda case: case.description, +) +def test_given_adapter_parameter_when_attaching_native_test_then_actual_sql_uses_adapter_literal( + test_case: AdapterParameterBoundaryTestCase, + tmp_path: Path, +) -> None: + adapters: dict[str, BaseAdapter] = { + "bigquery": BigQueryAdapter(), + "sqlserver": SqlServerAdapter(), + } + adapter: BaseAdapter = adapters[test_case.adapter_name] + write_files( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": f'name = "orders"\nadapter = "{test_case.adapter_name}"\n', + "models/raw_orders.sql": 'MODEL (description "Raw orders");\nSELECT 1 AS value\n', + "models/orders.sql": 'MODEL (description "Orders");\nSELECT value FROM __ref("raw_orders")\n', + "tests/unit/order_values.sql": ( + f"TEST (parameters (value {test_case.declaration}), cases (single (value {test_case.literal})));\n" + 'WITH __ref__raw_orders AS (SELECT @param("value") AS value), ' + '__expected__orders AS (SELECT @param("value") AS value) SELECT 1\n' + ), + }, + ) + inputs: CompileInputsView = build_compile_inputs( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=CollectionRendering.VALUE_LIST, + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + run_id="test_run", + ) + assert len(inputs.test_inputs) == 1 + assert test_case.expected_sql in inputs.test_inputs[0].sql_body + assert '@param("value")' not in inputs.test_inputs[0].sql_body + + +@pytest.mark.parametrize( + "test_case", + [ + QuotedHookBoundaryTestCase( + "quoted apostrophe and argument marker", "O'Brien@orders", "SELECT 'O''Brien@orders'" + ) + ], + ids=lambda case: case.description, +) +def test_given_quoted_hook_argument_when_attaching_then_inserted_marker_is_not_rescanned( + test_case: QuotedHookBoundaryTestCase, + tmp_path: Path, +) -> None: + write_files( + project_dir=tmp_path, + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "hooks/sql/notify.sql": "HOOK (description \"Record a label\");\nSELECT @'label'\n", + "models/orders.sql": ( + f'MODEL (description "Orders", pre_hooks [sql("notify", label: {json.dumps(test_case.argument)})]);\nSELECT 1 AS order_id\n' + ), + }, + ) + inputs: CompileInputsView = build_compile_inputs( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter_context=CompileAdapterContext( + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, + ), + run_id="test_run", + ) + hooks: list[SqlHookEntry] = cast( + list[SqlHookEntry], inputs.model_inputs[0].config.values["pre_hooks"] + ) + assert len(hooks) == 1 + assert hooks[0].statement.strip() == test_case.expected_sql diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_resource_diagnostics.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_resource_diagnostics.py new file mode 100644 index 0000000000..bcf36d13a1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_resource_diagnostics.py @@ -0,0 +1,161 @@ +"""P010, P011 and C216 computed by the native compiled project over its attached resources.""" + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + NativeResourceDiagnosticTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + cursor_model_files, + resource_diagnostics_of, +) + +_CURSOR_MESSAGE: str = ( + "incremental model 'daily_revenue' reads no inputs, so builds after the first " + "cannot work out their cursor window" +) +_UDF_AFTER_MENTIONS: str = ( + '"""Mentions @udf before the decorator."""\n' + "# @udf in a comment\n" + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "INTEGER"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(amount):\n" + " return amount > 100\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeResourceDiagnosticTestCase( + description="MODEL header after leading blank lines", + files={ + "models/order_totals.sql": "\n \t\n\n MODEL (materialized table);\nSELECT 1 AS id\n" + }, + expected_diagnostics=( + ( + "P010", + "model 'order_totals' has no description", + "models/order_totals.sql", + 4, + 5, + None, + ), + ), + ), + NativeResourceDiagnosticTestCase( + description="FUNCTION header after a leading blank line", + files={ + "functions/sql/is_large_order.sql": "\nFUNCTION (returns BOOLEAN);\n\nSELECT TRUE\n" + }, + expected_diagnostics=( + ( + "P010", + "function 'is_large_order' has no description", + "functions/sql/is_large_order.sql", + 2, + 1, + None, + ), + ), + ), + NativeResourceDiagnosticTestCase( + description="HOOK header on the first line", + files={"hooks/sql/analyze_orders.sql": "HOOK ();\nSELECT 1\n"}, + expected_diagnostics=( + ( + "P010", + "hook 'analyze_orders' has no description", + "hooks/sql/analyze_orders.sql", + 1, + 1, + None, + ), + ), + ), + NativeResourceDiagnosticTestCase( + description="@udf decorator after docstring and comment mentions", + files={"functions/python/is_large_order.py": _UDF_AFTER_MENTIONS}, + expected_diagnostics=( + ( + "P010", + "function 'is_large_order' has no description", + "functions/python/is_large_order.py", + 6, + 1, + None, + ), + ), + ), + NativeResourceDiagnosticTestCase( + description="second seed entry of its YAML file", + files={ + "seeds/product_labels.csv": "product_type_id\n1\n", + "seeds/product_types.csv": "product_type_id\n1\n", + "seeds/product_types.yml": ( + "seeds:\n - name: product_labels\n description: Labels\n columns:\n" + " - name: product_type_id\n type: INTEGER\n" + " - name: product_types\n columns:\n" + " - name: product_type_id\n type: INTEGER\n" + ), + }, + expected_diagnostics=( + ( + "P010", + "seed 'product_types' has no description", + "seeds/product_types.yml", + 7, + 1, + None, + ), + ), + ), + NativeResourceDiagnosticTestCase( + description="cursor model reading only constants is located at its cursor entry", + files=cursor_model_files(), + expected_diagnostics=( + ("P011", _CURSOR_MESSAGE, "models/finance/daily_revenue.sql", 5, 3, 22), + ), + ), + NativeResourceDiagnosticTestCase( + description="cursor model reading a ref", + files=cursor_model_files(read=' FROM __ref("payments")'), + expected_diagnostics=(), + ), + NativeResourceDiagnosticTestCase( + description="audit factory only one model attaches", + files={ + "models/orders.sql": ( + 'MODEL (description "Orders", audit_factories [quality]);\nSELECT 1 AS id\n' + ), + "python/factories/quality.py": ( + "from sqlbuild.audits import audit_factory\n\n\n" + "@audit_factory\ndef quality():\n return []\n\n\n" + "@audit_factory\ndef unused_quality():\n return []\n" + ), + }, + expected_diagnostics=( + ( + "C216", + "Audit factory 'unused_quality' is not referenced by any model and will " + "not produce audits", + "python/factories/quality.py", + 9, + 1, + None, + ), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_resources_when_compiling_then_native_owner_reports_their_diagnostics( + test_case: NativeResourceDiagnosticTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + diagnostics: tuple[tuple[object, ...], ...] = resource_diagnostics_of( + project_dir=tmp_path, files=test_case.files, monkeypatch=monkeypatch + ) + + assert diagnostics == test_case.expected_diagnostics diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py new file mode 100644 index 0000000000..d34587030e --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py @@ -0,0 +1,146 @@ +"""Sources attached by the native model loop and assembled into the native compiled project.""" + +from dataclasses import asdict +from pathlib import Path + +import pytest + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSourceView, SourceInput +from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + NativeSourceAttachmentTestCase, + NativeSourceErrorTestCase, +) +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + column_metadata, + compile_source_project, +) + +_TEMPLATED_SOURCE_YML: str = ( + "sources:\n" + " - name: raw_orders\n" + " database: lake_${region}\n" + " schema: raw_${region}\n" + " table: orders_${region}\n" + " description: Orders from ${region}.\n" + " meta:\n owner: team_${region}\n" + " columns:\n" + " - name: amount\n" + " type: DECIMAL\n" + " description: Amount in ${region}.\n" + " audits:\n" + " - accepted_values:\n" + " values: ['${region}']\n" +) +_EXPRESSION_SOURCE_YML: str = ( + "sources:\n" + " - name: open_orders\n" + " contract: none\n" + ' expression: "SELECT @cents(1) AS cents, @enum(\\"order_status\\").PLACED AS status,' + ' @const(\\"order_cap\\") AS cap, \'@@region\' AS region"\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeSourceAttachmentTestCase( + description="templated metadata, columns and audit arguments under the default contract", + source_yml=_TEMPLATED_SOURCE_YML, + source_name="raw_orders", + expected_entry={ + "database": "lake_eu", + "schema": "raw_eu", + "table": "orders_eu", + "description": "Orders from eu.", + "contract": "enforced", + "meta": {"owner": "team_eu"}, + "column_metadata": (("Amount in eu.", ({"values": ["eu"]},)),), + }, + expected_usages=(), + ), + NativeSourceAttachmentTestCase( + description="expression expanding macros, declarations and variables", + source_yml=_EXPRESSION_SOURCE_YML, + source_name="open_orders", + expected_entry={ + "expression": "SELECT 1 * 100 AS cents, 'PLACED' AS status, 3 AS cap," + " 'eu' AS region", + "contract": "none", + }, + expected_usages=("enum:order_status", "constant:order_cap", "macro:cents"), + ), + ], + ids=lambda case: case.description, +) +def test_given_source_when_compiling_then_native_project_holds_its_attached_entry( + test_case: NativeSourceAttachmentTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: CompiledProjectView + project, inputs = compile_source_project( + project_dir=tmp_path, source_yml=test_case.source_yml, monkeypatch=monkeypatch + ) + + source: CompiledSourceView = project.sources[0] + source_input: SourceInput = inputs.source_inputs[0] + entry: dict[str, object] = asdict(source.source_entry) + entry["column_metadata"] = tuple(map(column_metadata, source.source_entry.columns)) + assert isinstance(source, _native.NativeCompiledSource) + assert isinstance(source_input, _native.NativeSourceInput) + assert source.name == test_case.source_name + assert {key: entry[key] for key in test_case.expected_entry} == test_case.expected_entry + assert ( + tuple( + f"{usage.declaration.kind.value}:{usage.declaration.name}" + for usage in source_input.declaration_usages + ) + == test_case.expected_usages + ) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeSourceErrorTestCase( + description="unknown variable in column metadata", + source_yml=( + "sources:\n - name: raw_orders\n columns:\n - name: amount\n" + " meta:\n unit: ${currency}\n" + ), + expected_message=( + "source raw_orders column amount meta references unknown variable 'currency'" + ), + expects_help=False, + ), + NativeSourceErrorTestCase( + description="cursor intrinsic in an expression", + source_yml=( + "sources:\n - name: raw_orders\n" + ' expression: "SELECT 1 AS id WHERE id > __cursor_start()"\n' + ), + expected_message=( + "Source expression 'raw_orders' uses cursor intrinsics, which are only" + " supported in cursor-based incremental model query SQL" + ), + expects_help=False, + ), + NativeSourceErrorTestCase( + description="expression that does not parse as a relation", + source_yml='sources:\n - name: raw_orders\n expression: "SELECT FROM WHERE"\n', + expected_message="SQL syntax error in source expression 'raw_orders'", + expects_help=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_source_when_compiling_then_native_attachment_rejects_it( + test_case: NativeSourceErrorTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with pytest.raises(CompileInputError) as raised: + _ = compile_source_project( + project_dir=tmp_path, source_yml=test_case.source_yml, monkeypatch=monkeypatch + ) + + assert test_case.expected_message in str(raised.value) + assert (raised.value.help is not None) is test_case.expects_help diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index 4d735dffe3..260e23899d 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -2,27 +2,29 @@ from __future__ import annotations -import itertools import random from collections import Counter -from collections.abc import Callable, Iterator, Sequence +from collections.abc import Callable, Sequence from dataclasses import replace from pathlib import Path -from typing import Any, NamedTuple +from typing import Any, NamedTuple, TypeGuard import pytest -import sqlbuild._native as native_module +import sqlbuild.compiler.contracts.main._evaluate_native_model_contracts as contract_entry +from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, CompilerDiagnostic, DynamicColumnContractProof, DynamicColumnFamilyProof, InferredColumn, ) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts +from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.pipeline.main.graph import build_project_graph @@ -161,11 +163,10 @@ type ContractView = tuple[CompilerDiagnostic, ...] type NativeContractRequest = tuple[str, bool, list[Any]] -type NativeContractOutcome = list[Any] _EMPTY_SCHEMA_ROW: tuple[list[Any], list[Any], bool, bool] = ([], [], False, False) -def compiled_contract_project(*, project_dir: Path) -> CompiledProject: +def compiled_contract_project(*, project_dir: Path) -> CompiledProjectView: """Write and compile the base project whose models the generators perturb.""" files: dict[str, str] = { @@ -183,11 +184,12 @@ def compiled_contract_project(*, project_dir: Path) -> CompiledProject: ).project -def perturbed_project(*, project: CompiledProject, rng: random.Random) -> CompiledProject: +def perturbed_project(*, project: CompiledProjectView, rng: random.Random) -> CompiledProjectView: """The project with every model's contract inputs replaced by generated ones.""" return replace( - project, + compiled_project_record(project), + native_project=None, settings=replace( project.settings, column_contract_mode=rng.choice(tuple(ColumnContractMode)), @@ -197,7 +199,7 @@ def perturbed_project(*, project: CompiledProject, rng: random.Random) -> Compil ) -def perturbed_model(*, model: CompiledModel, rng: random.Random) -> CompiledModel: +def perturbed_model(*, model: CompiledModelView, rng: random.Random) -> CompiledModelView: """One model with generated contract config, declared shape, inference and proof.""" values: dict[str, object] = dict(model.config.values) @@ -208,7 +210,7 @@ def perturbed_model(*, model: CompiledModel, rng: random.Random) -> CompiledMode ): values.update(rng.choice(({}, *({key: option} for option in options)))) return replace( - model, + compiled_model_record(model), config=replace(model.config, values=values), schema_entry=rng.choice((None, *(_schema_entry(model=model, rng=rng),) * 9)), inferred_columns=rng.choice((None, *(_inferred_columns(rng=rng),) * 9)), @@ -218,7 +220,7 @@ def perturbed_model(*, model: CompiledModel, rng: random.Random) -> CompiledMode ) -def _schema_entry(*, model: CompiledModel, rng: random.Random) -> SchemaModelEntry: +def _schema_entry(*, model: CompiledModelView, rng: random.Random) -> SchemaModelEntry: columns: tuple[SchemaColumn, ...] = tuple( SchemaColumn( name=name, @@ -278,7 +280,7 @@ def _dynamic_proof(*, rng: random.Random) -> DynamicColumnContractProof: ) -def contract_diagnostics(*, project: CompiledProject, dialect: str | None) -> ContractView: +def contract_diagnostics(*, project: CompiledProjectView, dialect: str | None) -> ContractView: """The project's contract diagnostics, through the public contract entry point.""" return evaluate_model_contracts(project=project, dialect=dialect).diagnostics @@ -316,24 +318,63 @@ def record_native_outcomes(*, monkeypatch: pytest.MonkeyPatch) -> NativeContract """Record native contract outcomes for the rest of the test.""" record: NativeContractRecord = NativeContractRecord(statuses=Counter(), codes=Counter()) - evaluate: Callable[..., list[NativeContractOutcome]] = ( - native_module.evaluate_native_model_contracts - ) + evaluate: Callable[..., ContractValidationResult] = contract_entry.native_model_contracts - def counted(request: NativeContractRequest) -> list[NativeContractOutcome]: - outcomes: list[NativeContractOutcome] = evaluate(request) - record.statuses.update(native_contract_statuses(request=request, outcomes=outcomes)) - rows: Iterator[Sequence[Any]] = itertools.chain.from_iterable(outcomes) - record.codes.update(row[0] for row in rows) - return outcomes + def counted( + *, project: CompiledProjectView, dialect: TypeDialect | str | None + ) -> ContractValidationResult: + result: ContractValidationResult = evaluate(project=project, dialect=dialect) + record.statuses.update(contract_statuses(project=project, result=result)) + record.codes.update(diagnostic.code for diagnostic in result.diagnostics) + return result - monkeypatch.setattr(native_module, "evaluate_native_model_contracts", counted) + monkeypatch.setattr(contract_entry, "native_model_contracts", counted) return record -def native_contract_statuses( - *, request: NativeContractRequest, outcomes: list[NativeContractOutcome] +def contract_statuses( + *, project: CompiledProjectView, result: ContractValidationResult ) -> Counter[str]: + """Counts of evaluated models, typed comparisons and diagnostics for one contract run.""" + + request: NativeContractRequest = ( + "", + project.settings.column_contract_mode == ColumnContractMode.IMPLICIT, + [_status_row(model) for model in project.models], + ) + return native_contract_statuses(request=request, diagnostics=len(result.diagnostics)) + + +def _status_row(model: CompiledModelView) -> tuple[object, ...]: + contract: object = model.config.values.get("contract") + return ( + model.name, + (None, contract)[isinstance(contract, str)], + next(map(_status_schema, filter(_present, (model.schema_entry,))), None), + next(map(_status_columns, filter(_present, (model.inferred_columns,))), None), + ) + + +def _present[T](value: T | None) -> TypeGuard[T]: + return value is not None + + +def _status_columns(columns: tuple[InferredColumn, ...]) -> list[tuple[str, str | None, bool]]: + return [ + (column.name, column.type, column.nullability == InferredNullability.NULLABLE) + for column in columns + ] + + +def _status_schema(schema: SchemaModelEntry) -> tuple[object, ...]: + return ( + [(column.name, column.type, column.nullable is False, False) for column in schema.columns], + [(family.name, family.type) for family in schema.dynamic_columns], + bool(schema.type_enforcement), + ) + + +def native_contract_statuses(*, request: NativeContractRequest, diagnostics: int) -> Counter[str]: """Per-request counts: `native` for evaluated models, `typed_comparisons` for those that compared at least one typed column, and `native_diagnostics`.""" @@ -346,7 +387,7 @@ def native_contract_statuses( statuses["typed_comparisons"] += sum( _compares_typed_column(model=model, implicit=implicit) for model in evaluated ) - statuses["native_diagnostics"] += sum(len(diagnostics) for diagnostics in outcomes) + statuses["native_diagnostics"] += diagnostics return statuses @@ -376,16 +417,20 @@ def _compares_typed_column(*, model: Sequence[Any], implicit: bool) -> bool: ) -def with_declared_type(*, project: CompiledProject, declared_type: str) -> CompiledProject: +def with_declared_type(*, project: CompiledProjectView, declared_type: str) -> CompiledProjectView: """The project with its enforced staging model declaring `declared_type` for every column.""" - by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} - staging: CompiledModel = by_name[_ENFORCED_MODEL] + by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} + staging: CompiledModelView = by_name[_ENFORCED_MODEL] assert staging.schema_entry is not None columns: tuple[SchemaColumn, ...] = tuple( replace(column, type=declared_type) for column in staging.schema_entry.columns ) by_name[_ENFORCED_MODEL] = replace( - staging, schema_entry=replace(staging.schema_entry, columns=columns) + compiled_model_record(staging), schema_entry=replace(staging.schema_entry, columns=columns) + ) + return replace( + compiled_project_record(project), + native_project=None, + models=tuple(by_name[model.name] for model in project.models), ) - return replace(project, models=tuple(by_name[model.name] for model in project.models)) diff --git a/tests/integration/src/sqlbuild/compiler/contracts/test_native_model_contracts.py b/tests/integration/src/sqlbuild/compiler/contracts/test_native_model_contracts.py index 2e21d0a4f0..f7e8c93063 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/test_native_model_contracts.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/test_native_model_contracts.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.contracts._test_types import ( FormerlyDeferredContractTestCase, GeneratedContractTestCase, @@ -56,10 +56,10 @@ def test_given_generated_contracts_when_validating_then_every_contract_family_is rng: random.Random = random.Random(test_case.seed) record: NativeContractRecord = record_native_outcomes(monkeypatch=monkeypatch) project_dir: Path = tmp_path / "project" - base: CompiledProject = compiled_contract_project(project_dir=project_dir) + base: CompiledProjectView = compiled_contract_project(project_dir=project_dir) golden: list[GoldenEntry] = [] for _ in range(test_case.variants): - project: CompiledProject = perturbed_project(project=base, rng=rng) + project: CompiledProjectView = perturbed_project(project=base, rng=rng) golden.append( golden_entry( "contracts", @@ -164,7 +164,7 @@ def test_given_generated_contracts_when_validating_then_every_contract_family_is def test_given_input_python_once_answered_when_validating_then_native_diagnostics_are_returned( test_case: FormerlyDeferredContractTestCase, tmp_path: Path ) -> None: - project: CompiledProject = with_declared_type( + project: CompiledProjectView = with_declared_type( project=compiled_contract_project(project_dir=tmp_path / "project"), declared_type=test_case.declared_type, ) @@ -194,7 +194,7 @@ def test_given_input_python_once_answered_when_validating_then_native_diagnostic def test_given_unknown_dialect_when_validating_typed_contracts_then_python_error_is_raised( test_case: UnknownDialectContractTestCase, tmp_path: Path ) -> None: - project: CompiledProject = compiled_contract_project(project_dir=tmp_path / "project") + project: CompiledProjectView = compiled_contract_project(project_dir=tmp_path / "project") with pytest.raises(ValueError, match=test_case.expected_error): _ = contract_diagnostics(project=project, dialect=test_case.dialect) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py index 64308bde13..66d5db1cb5 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/helpers.py @@ -38,7 +38,6 @@ native_payload_error, native_text_runtime, ) -from sqlbuild.compiler.discovery._helpers.yml.project import load_local_config, load_project_config from sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer import ( SelectedContractInputDiscoverer, ) @@ -61,7 +60,6 @@ ) from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig from sqlbuild.sql_values.models import AuthoredSqlSet, AuthoredSqlValueCall from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( GeneratedDeclarationFileTestCase, @@ -575,46 +573,6 @@ def repository_files(*, pattern: str) -> list[Path]: ) -def _python_project_fields(project: ProjectConfig, local: LocalConfig) -> dict[str, object]: - return { - "name": project.name, - "adapter": project.adapter, - "default_target": project.default_target, - "sql_analysis": project.settings.sql_analysis, - "require_sql_analysis": project.settings.require_sql_analysis, - "enforce_placement": project.scopes.enforce_placement, - "enforce_explicit_references": project.references.enforce_explicit, - "vars": [list(item) for item in project.vars.items()], - "path_default_keys": list(project.path_defaults), - "target_names": list(project.targets), - "local_target": local.target, - "local_adapter": local.adapter, - "local_sql_analysis": {True: local.settings.sql_analysis, False: None}[ - "sql_analysis" in local.setting_overrides - ], - "local_vars": [list(item) for item in local.vars.items()], - "local_target_names": list(local.targets), - } - - -def python_project_outcome(*, project_dir: Path) -> object: - """Return the discovery fields Python's loaders read, or the rejection marker.""" - - outcome: list[object] = [PYTHON_ERROR] - with suppress(Exception): - outcome[0] = _python_project_fields( - load_project_config(project_dir=project_dir), load_local_config(project_dir=project_dir) - ) - return outcome[0] - - -def native_project_outcome(*, project_dir: Path) -> object: - """Return the discovery fields the native reader reads, or its failure marker.""" - - parsed: dict[str, object] = json.loads(_native._oracle_project_config(str(project_dir))) - return {True: DEFERRED, False: parsed}["error" in parsed] - - def deferred_valid_count(*, expected: list[object], actual: list[object]) -> int: """Return how many documents Python accepts but the native reader defers back to Python.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py deleted file mode 100644 index 243dd3840b..0000000000 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_config_oracle.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Compare the native project configuration reader with SQLBuild's Python loaders.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( - RepositoryFileOracleTestCase, -) -from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( - deferred_valid_count, - harmful_mismatches, - native_project_outcome, - python_project_outcome, - repository_files, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RepositoryFileOracleTestCase( - description="fixture and example projects", - pattern="**/sqlbuild_project.toml", - expected_minimum_files=5, - ), - ], - ids=lambda case: case.description, -) -def test_given_repository_projects_when_reading_natively_then_discovery_fields_match_python( - test_case: RepositoryFileOracleTestCase, -) -> None: - project_dirs: list[Path] = [path.parent for path in repository_files(pattern=test_case.pattern)] - - expected: list[object] = [python_project_outcome(project_dir=path) for path in project_dirs] - actual: list[object] = [native_project_outcome(project_dir=path) for path in project_dirs] - - assert len(project_dirs) >= test_case.expected_minimum_files - assert deferred_valid_count(expected=expected, actual=actual) == test_case.expected_deferred - assert harmful_mismatches( - inputs=list(map(str, project_dirs)), expected=expected, actual=actual - ) == list(test_case.expected_mismatches) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py index f44563fea8..70ecf9ea58 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_toml_oracle.py @@ -34,7 +34,7 @@ generator="toml", seed=51, case_count=5000, - expected_maximum_deferred=350, + expected_maximum_deferred=0, ), GeneratedDocumentOracleTestCase( description="tables, arrays of tables, inline tables and dotted keys reopening namespaces", diff --git a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py index 79e719e046..4765de1b71 100644 --- a/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/explicit_references/helpers.py @@ -12,10 +12,10 @@ from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, - CompiledProject, CompilerDiagnostic, LiteralSqlRelation, ) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.sql_values.types import CollectionRendering @@ -369,7 +369,7 @@ def compile_outcome(*, project_dir: Path) -> tuple[object, ...]: """Assemble the project: every diagnostic and unmatched relation, or the error raised.""" try: - project: CompiledProject = assemble_project( + project: CompiledProjectView = assemble_project( inputs=build_compile_inputs( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter_context=_ADAPTER_CONTEXT, diff --git a/tests/integration/src/sqlbuild/compiler/graph/__init__.py b/tests/integration/src/sqlbuild/compiler/graph/__init__.py new file mode 100644 index 0000000000..0d03ff56d4 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/graph/__init__.py @@ -0,0 +1 @@ +"""Native project graph integration tests.""" diff --git a/tests/integration/src/sqlbuild/compiler/graph/_test_types.py b/tests/integration/src/sqlbuild/compiler/graph/_test_types.py new file mode 100644 index 0000000000..c51c65739c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/graph/_test_types.py @@ -0,0 +1,24 @@ +"""Test case types for native project graph integration tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeGraphSelectionTestCase: + """A real compile selection resolved on the project's native graph.""" + + description: str + select: tuple[str, ...] + expected_models: frozenset[str] + expected_graph_builds: int + + +@dataclass(frozen=True) +class NativeGraphSelectorErrorTestCase: + """A compile selector the native grammar rejects with the planner's error.""" + + description: str + select: str + expected_error: str diff --git a/tests/integration/src/sqlbuild/compiler/graph/helpers.py b/tests/integration/src/sqlbuild/compiler/graph/helpers.py new file mode 100644 index 0000000000..bb647d0aac --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/graph/helpers.py @@ -0,0 +1,53 @@ +"""Helpers for native project graph integration tests.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import sqlbuild._native as native_module + +ORDERS_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/staging/stg_orders.sql": ( + "MODEL (description 'Staged orders.', tags [daily]);\nSELECT 1 AS order_id\n" + ), + "models/marts/orders.sql": ( + "MODEL (description 'Orders.');\nSELECT order_id FROM __ref(\"stg_orders\")\n" + ), + "models/marts/customers.sql": "MODEL (description 'Customers.');\nSELECT 1 AS customer_id\n", +} +MODEL_NAMES: frozenset[str] = frozenset({"stg_orders", "orders", "customers"}) + + +def write_files(*, project_dir: Path, files: dict[str, str]) -> None: + """Write project files under `project_dir`.""" + + for relative_path, text in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def record_compile_owners( + *, monkeypatch: pytest.MonkeyPatch +) -> list[native_module.NativeCompiledProject]: + """Record every native compiled project the compile creates, to read its graph builds.""" + + owners: list[native_module.NativeCompiledProject] = [] + owner_type: type[native_module.NativeCompiledProject] = native_module.NativeCompiledProject + + def recorded() -> native_module.NativeCompiledProject: + owner: native_module.NativeCompiledProject = owner_type() + owners.append(owner) + return owner + + monkeypatch.setattr(native_module, "NativeCompiledProject", recorded) + return owners + + +def compiled_models(*, stdout: str) -> dict[str, bool]: + """Whether the compile report lists each project model.""" + + return {name: f"── {name} " in stdout for name in sorted(MODEL_NAMES)} diff --git a/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py b/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py new file mode 100644 index 0000000000..3efb4be77f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py @@ -0,0 +1,107 @@ +"""Compile selection on the native project graph built from the compiled project.""" + +from pathlib import Path + +import pytest + +import sqlbuild._native as native_module +from sqlbuild.cli.entry.main.entry import main +from tests.integration.src.sqlbuild.compiler.graph._test_types import ( + NativeGraphSelectionTestCase, + NativeGraphSelectorErrorTestCase, +) +from tests.integration.src.sqlbuild.compiler.graph.helpers import ( + MODEL_NAMES, + ORDERS_PROJECT_FILES, + compiled_models, + record_compile_owners, + write_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeGraphSelectionTestCase( + description="no selection builds one graph for the compiled project", + select=(), + expected_models=frozenset({"stg_orders", "orders", "customers"}), + expected_graph_builds=1, + ), + NativeGraphSelectionTestCase( + description="tag with downstream expansion", + select=("--select", "tag:daily+"), + expected_models=frozenset({"stg_orders", "orders"}), + expected_graph_builds=2, + ), + NativeGraphSelectionTestCase( + description="models-rooted folder", + select=("--select", "models/marts"), + expected_models=frozenset({"orders", "customers"}), + expected_graph_builds=2, + ), + NativeGraphSelectionTestCase( + description="name glob minus an exclusion", + select=("--select", "*orders", "--exclude", "stg_orders"), + expected_models=frozenset({"orders"}), + expected_graph_builds=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_compile_selection_when_compiling_then_native_graph_selects_models( + test_case: NativeGraphSelectionTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_files(project_dir=tmp_path, files=ORDERS_PROJECT_FILES) + owners: list[native_module.NativeCompiledProject] = record_compile_owners( + monkeypatch=monkeypatch + ) + + exit_code: int = main( + ["--project-dir", str(tmp_path), "--no-color", "compile", "--no-cache", *test_case.select] + ) + + assert exit_code == 0 + assert compiled_models(stdout=capsys.readouterr().out) == { + name: name in test_case.expected_models for name in sorted(MODEL_NAMES) + } + assert sum(owner.graph_builds() for owner in owners) == test_case.expected_graph_builds + + +@pytest.mark.parametrize( + "test_case", + [ + NativeGraphSelectorErrorTestCase( + description="unknown name suggests the nearest model", + select="ordrs", + expected_error="error[S007]: unknown selector name 'ordrs'", + ), + NativeGraphSelectorErrorTestCase( + description="folder outside the models root", + select="marts/", + expected_error="error[S012]: path selectors require an explicit root", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_compile_selector_when_compiling_then_planner_error_is_reported( + test_case: NativeGraphSelectorErrorTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_files(project_dir=tmp_path, files=ORDERS_PROJECT_FILES) + + exit_code: int = main( + ["--project-dir", str(tmp_path), "--no-color", "compile", "--select", test_case.select] + ) + output: str = "".join(capsys.readouterr()) + + assert exit_code == 1 + assert test_case.expected_error in output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py index bcc175b0bd..18eb9eee94 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py @@ -12,7 +12,9 @@ import sqlbuild._native as native_module from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( build_native_column_lineage, @@ -208,7 +210,7 @@ def generated_lineage_files(*, rng: random.Random, model_count: int) -> dict[str return files -def compiled_project(*, project_dir: Path, files: dict[str, str]) -> CompiledProject: +def compiled_project(*, project_dir: Path, files: dict[str, str]) -> CompiledProjectView: """Write and compile a project, keeping any diagnostics.""" for relative_path, contents in files.items(): @@ -221,18 +223,21 @@ def compiled_project(*, project_dir: Path, files: dict[str, str]) -> CompiledPro ).project -def without_compact_facts(project: CompiledProject) -> CompiledProject: +def without_compact_facts(project: CompiledProjectView) -> CompiledProjectView: """The project with every model sent through the parse fallback.""" - return replace( - project, - models=tuple(replace(model, fast_lineage_columns=None) for model in project.models), + return project_with( + project=project, + models=tuple( + replace(compiled_model_record(model), fast_lineage_columns=None) + for model in project.models + ), ) def lineage_views( *, - project: CompiledProject, + project: CompiledProjectView, dialect: str | None, model_names: frozenset[str] | None, ) -> tuple[list[object], list[object]]: @@ -246,7 +251,7 @@ def lineage_views( def _public_views( - *, project: CompiledProject, lineage: ProjectColumnLineage + *, project: CompiledProjectView, lineage: ProjectColumnLineage ) -> tuple[list[object], list[object]]: names: list[object] = [] values: list[object] = [] diff --git a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py index fd01d583e1..e9e42c653f 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py @@ -11,7 +11,9 @@ import pytest import sqlbuild._native as native_module -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( build_native_column_lineage, ) @@ -69,7 +71,7 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje differences: list[tuple[object, object, object]] = [] golden: list[GoldenEntry] = [] for index in range(test_case.count): - project: CompiledProject = compiled_project( + project: CompiledProjectView = compiled_project( project_dir=tmp_path / f"project_{index}", files=generated_lineage_files(rng=rng, model_count=_MODEL_COUNT), ) @@ -94,7 +96,7 @@ def test_given_generated_projects_when_building_fast_lineage_then_restored_proje ) ) _, restored = lineage_views( - project=replace(variant, binding_catalog=None), + project=project_with(project=variant, binding_catalog=None), dialect=dialect, model_names=model_names, ) @@ -140,14 +142,18 @@ def test_given_unparsable_model_when_building_fast_lineage_then_logs_and_omits_i caplog: pytest.LogCaptureFixture, ) -> None: statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) - project: CompiledProject = compiled_project( + project: CompiledProjectView = compiled_project( project_dir=tmp_path / "project", files=generated_lineage_files(rng=random.Random(11), model_count=_MODEL_COUNT), ) - broken: CompiledProject = replace( - project, + broken: CompiledProjectView = project_with( + project=project, models=( - replace(project.models[0], query_sql=test_case.query_sql, fast_lineage_columns=None), + replace( + compiled_model_record(project.models[0]), + query_sql=test_case.query_sql, + fast_lineage_columns=None, + ), *project.models[1:], ), ) @@ -192,14 +198,14 @@ def test_given_formerly_deferred_models_when_building_fast_lineage_then_native_a record_dir: Path = tmp_path / "records" monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) - compiled: CompiledProject = without_compact_facts( + compiled: CompiledProjectView = without_compact_facts( compiled_project( project_dir=tmp_path / "project", files=generated_lineage_files(rng=random.Random(7), model_count=_MODEL_COUNT), ) ) - project: CompiledProject = replace( - compiled, binding_catalog=(None, compiled.binding_catalog)[test_case.keeps_catalog] + project: CompiledProjectView = project_with( + project=compiled, binding_catalog=(None, compiled.binding_catalog)[test_case.keeps_catalog] ) _ = lineage_views(project=project, dialect=test_case.dialect, model_names=None) @@ -229,14 +235,14 @@ def test_given_unknown_dialect_when_building_fast_lineage_then_python_error_rais record_dir: Path = tmp_path / "records" monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) statuses: Counter[str] = record_native_outcomes(monkeypatch=monkeypatch) - compiled: CompiledProject = without_compact_facts( + compiled: CompiledProjectView = without_compact_facts( compiled_project( project_dir=tmp_path / "project", files=generated_lineage_files(rng=random.Random(7), model_count=_MODEL_COUNT), ) ) - project: CompiledProject = replace( - compiled, binding_catalog=(None, compiled.binding_catalog)[test_case.keeps_catalog] + project: CompiledProjectView = project_with( + project=compiled, binding_catalog=(None, compiled.binding_catalog)[test_case.keeps_catalog] ) with pytest.raises(ValueError, match=test_case.expected_error): @@ -264,14 +270,18 @@ def test_given_unknown_dialect_when_building_fast_lineage_then_python_error_rais def test_given_parser_panic_when_building_fast_lineage_then_raises_native_compiler_error( test_case: ParserPanicLineageTestCase, tmp_path: Path ) -> None: - project: CompiledProject = compiled_project( + project: CompiledProjectView = compiled_project( project_dir=tmp_path / "project", files=generated_lineage_files(rng=random.Random(11), model_count=_MODEL_COUNT), ) - broken: CompiledProject = replace( - project, + broken: CompiledProjectView = project_with( + project=project, models=( - replace(project.models[0], query_sql=test_case.query_sql, fast_lineage_columns=None), + replace( + compiled_model_record(project.models[0]), + query_sql=test_case.query_sql, + fast_lineage_columns=None, + ), *project.models[1:], ), ) diff --git a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py index 76a39990b0..79f3589688 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/_test_types.py @@ -14,6 +14,7 @@ class HeaderMetadataParityTestCase: count: int expected_minimum_parsed: int expected_minimum_rejected: int + expected_maximum_later_validated: int @dataclass(frozen=True) @@ -23,3 +24,13 @@ class ModelConfigTierTestCase: description: str engine: str expected_native_calls: dict[str, int] + + +@dataclass(frozen=True) +class ColumnMetadataExpansionTestCase: + """A header column field holding a config template or macro, and the loop's outcome.""" + + description: str + header_column: str + expected_type: str | None + expected_error: str | None diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 41ce3e6abb..7e5626dbc8 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -3,10 +3,11 @@ from __future__ import annotations import random -from collections.abc import Callable -from dataclasses import dataclass, fields +import re +from collections.abc import Callable, Iterator +from dataclasses import dataclass, replace from pathlib import Path -from typing import cast +from typing import TypeGuard, cast import pytest @@ -15,21 +16,16 @@ from sqlbuild.compiler.auditing.main._parse_audit_instances import parse_audit_instances from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import CompileAdapterContext, CompileProjectInputs +from sqlbuild.compiler.compile.models import CompileAdapterContext +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main._model_schema_columns import parse_schema_columns from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.model_config.main._parse_native_header_metadata import ( - parse_native_header_metadata, -) -from sqlbuild.compiler.model_config.models import NativeHeaderMetadata from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( resolve_effective_collection_rendering, ) -from sqlbuild.spec.contracts.models import SourceLocation -from tests.integration.src.sqlbuild.compiler.helpers import mismatches +from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation _TEXTS: tuple[object, ...] = ( "INTEGER", @@ -38,10 +34,6 @@ "", " ", "\u00a0", - "${target.schema}", - "@cents(amount)", - "@cents\u2003(amount)", - "@cents\x1f(amount)", "mail@example.com", "\x1c", 1, @@ -119,10 +111,7 @@ ) -NATIVE_MODEL_CONFIG_ENTRIES: tuple[str, ...] = ( - "parse_model_header_metadata", - "expand_config_templates", -) +NATIVE_MODEL_CONFIG_ENTRIES: tuple[str, ...] = ("expand_config_templates",) _NATIVE_ENTRIES: dict[str, Callable[..., object]] = { name: getattr(_native, name) for name in NATIVE_MODEL_CONFIG_ENTRIES } @@ -149,13 +138,36 @@ } +LATER_VALIDATION_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"is missing argument '[^']+' for generic audit '[^']+'$"), + re.compile(r"column '[^']+' migrate_from must name one column of the model's existing table$"), + re.compile( + r"audit '[^']+': thresholds and minimum_samples are only valid for measurement audits$" + ), +) +"""Validations the loop runs after parsing a header, which the YAML schema parsers never run.""" +_PARITY_PROJECT: str = 'name = "orders"\nadapter = "duckdb"\n' +_PARITY_FILES: dict[str, str] = { + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized incremental, incremental_strategy append);\n" + "SELECT 1 AS order_id\n" + ), + **{ + f"audits/generic/{name}.sql": 'AUDIT ();\nSELECT * FROM __ref("@model") WHERE FALSE\n' + for name in ("a",) + }, +} + + @dataclass(frozen=True) class HeaderMetadataParity: - """How the native header metadata parse compared with Python's over one corpus.""" + """How the model loop's header attachment compared with the YAML parsers over one corpus; + the loop's later validation may reject a header the parsers accept.""" mismatches: list[tuple[object, object, object]] parsed: int rejected: int + later_validated: int def generated_header_metadata(*, rng: random.Random, count: int) -> list[tuple[object, object]]: @@ -179,7 +191,7 @@ def model_config_engine_outcome( discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine) adapter: DuckDbAdapter = DuckDbAdapter() - inputs: CompileProjectInputs = build_compile_inputs( + inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=CompileAdapterContext( value_renderer=adapter, @@ -223,35 +235,93 @@ def counted(*args: object) -> object: return counted -def header_metadata_parity(*, headers: list[tuple[object, object]]) -> HeaderMetadataParity: - """Compare the native parse or rejection with the YAML schema parsers Python still owns.""" +def header_metadata_parity( + *, project_dir: Path, headers: list[tuple[object, object]] +) -> HeaderMetadataParity: + """Compare each header the model loop attaches or rejects with the YAML schema parsers.""" + _ = (project_dir / "sqlbuild_project.toml").write_text(_PARITY_PROJECT, encoding="utf-8") + for relative_path, contents in _PARITY_FILES.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) model_files: list[DiscoveredSqlModelFile] = [ - _model_file(index=index, columns=columns, audits=audits) + _model_file(base=discovered.model_files[0], index=index, columns=columns, audits=audits) for index, (columns, audits) in enumerate(headers) ] - native: list[NativeHeaderMetadata] = [ - parse_native_header_metadata( - raw_columns=model_file.header_values.get("columns"), - raw_audits=model_file.header_values.get("audits"), - column_locations=model_file.header_column_locations, - file_path=model_file.relative_path, - ) - for model_file in model_files + actual: list[object] = [ + _loop_shape(discovered=discovered, model_file=model_file) for model_file in model_files ] + expected: list[object] = [ + _shape(_python_metadata(model_file=model_file)) for model_file in model_files + ] + outcomes: list[tuple[object, object, object]] = list( + zip([model_file.header_values for model_file in model_files], expected, actual, strict=True) + ) + rejected: int = sum(isinstance(shape, list) for shape in actual) return HeaderMetadataParity( - mismatches=mismatches( - inputs=[model_file.header_values for model_file in model_files], - expected=[ - _shape(_python_metadata(model_file=model_file)) for model_file in model_files - ], - actual=[_native_shape(metadata) for metadata in native], + mismatches=list(filter(_unexplained, outcomes))[:3], + parsed=len(actual) - rejected, + rejected=rejected, + later_validated=sum( + _later_validation(expected=expected, actual=shape) for _, expected, shape in outcomes ), - parsed=sum(_native_error(metadata) is None for metadata in native), - rejected=sum(_native_error(metadata) is not None for metadata in native), ) +def _unexplained(outcome: tuple[object, object, object]) -> bool: + """A differing outcome other than a parsed header a known later loop validation rejects.""" + + _, expected, actual = outcome + return expected != actual and not _later_validation(expected=expected, actual=actual) + + +def _later_validation(*, expected: object, actual: object) -> bool: + """Whether the parsers accepted the header and the loop rejected it with a known later check.""" + + return ( + isinstance(expected, tuple) + and isinstance(actual, list) + and any(pattern.search(str(actual[1])) for pattern in LATER_VALIDATION_PATTERNS) + ) + + +def _loop_shape( + *, discovered: DiscoveredProjectInputs, model_file: DiscoveredSqlModelFile +) -> object: + """The model loop's attached columns and audits, or the error it raised.""" + + try: + inputs: CompileInputsView = build_compile_inputs( + discovered_inputs=replace(discovered, model_files=(model_file,)), + adapter_context=CompileAdapterContext( + value_renderer=DuckDbAdapter(), + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, + ), + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, + ) + except Exception as error: # noqa: BLE001 - the exact loop outcome, whatever it is + return list(error_shape(error)[:4]) + entry: SchemaModelEntry | None = inputs.model_inputs[0].schema_entry + return _shape(_entry_metadata(entry)) + + +def _entry_metadata(entry: SchemaModelEntry | None) -> tuple[object, object]: + entries: Iterator[SchemaModelEntry] = filter(_attached, (entry,)) + return next(((item.columns, item.audits) for item in entries), ((), ())) + + +def _attached(entry: SchemaModelEntry | None) -> TypeGuard[SchemaModelEntry]: + return entry is not None + + def _python_metadata(*, model_file: DiscoveredSqlModelFile) -> object: try: return ( @@ -287,19 +357,6 @@ def _metadata_shape(value: object) -> object: return (repr(value), _attribute_orders(value), _locations(value)) -def _native_shape(parsed: NativeHeaderMetadata) -> object: - error: _native.NativeConfigError | None = _native_error(parsed) - shapes: list[object] = [ - list(error_shape(native_config_error(error=cast(_native.NativeConfigError, error)))[:4]) - for _ in range(error is not None) - ] - return (*shapes, _shape((parsed.columns, parsed.audits)))[0] - - -def _native_error(metadata: NativeHeaderMetadata) -> _native.NativeConfigError | None: - return metadata.columns_error or metadata.audits_error - - _SHAPES: dict[type, Callable[[object], object]] = {tuple: _metadata_shape} @@ -330,24 +387,19 @@ def _flattened(value: object) -> tuple[object, ...]: return (*columns, *audits) -def _model_file(*, index: int, columns: object, audits: object) -> DiscoveredSqlModelFile: - relative_path: Path = Path("models") / f"model_{index}.sql" +def _model_file( + *, base: DiscoveredSqlModelFile, index: int, columns: object, audits: object +) -> DiscoveredSqlModelFile: names: list[str] = [*filter(_is_text, getattr(columns, "keys", tuple)())] locations: dict[str, SourceLocation] = { - name: SourceLocation(path=relative_path, line=line + 2, column=3) + name: SourceLocation(path=base.relative_path, line=line + 2, column=3) for line, name in enumerate(names[: len(names) * (index % 2)]) } - model_file: DiscoveredSqlModelFile = object.__new__(DiscoveredSqlModelFile) - values: dict[str, object] = {field.name: None for field in fields(DiscoveredSqlModelFile)} - values.update( - file_path=Path("/project") / relative_path, - relative_path=relative_path, - header_values={"columns": columns, "audits": audits}, + return replace( + base, + header_values={**base.header_values, "columns": columns, "audits": audits}, header_column_locations=locations, ) - for name, value in values.items(): - object.__setattr__(model_file, name, value) - return model_file def _is_text(value: object) -> bool: @@ -424,3 +476,44 @@ def _audit(*, rng: random.Random) -> object: ), weights=(35, 10, 5, 50), )[0] + + +def column_metadata_outcome(*, project_dir: Path, header_column: str) -> tuple[object, object]: + """Compile a one-model project whose header declares `header_column`; return the attached + column's type, or the error the loop raised.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[vars]\ncolumn_type = "BIGINT"\n' + ), + "macros/amounts.py": 'def cents(column: str) -> str:\n return f"{column} * 100"\n', + "models/orders.sql": ( + f"MODEL (description 'Orders.', columns ({header_column}));\nSELECT 1 AS amount\n" + ), + } + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) + try: + inputs: CompileInputsView = build_compile_inputs( + discovered_inputs=discovered, + adapter_context=CompileAdapterContext( + value_renderer=DuckDbAdapter(), + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, declaration_override=None + ), + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, + ), + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, + ) + except Exception as error: # noqa: BLE001 - the exact loop outcome, whatever it is + return None, str(error) + columns: tuple[object, ...] = cast( + tuple[object, ...], _entry_metadata(inputs.model_inputs[0].schema_entry)[0] + ) + return getattr(columns[0], "type", None), None diff --git a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py index 7e88a5701c..7d49a3444d 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/test_native_model_config.py @@ -8,11 +8,13 @@ import pytest from tests.integration.src.sqlbuild.compiler.model_config._test_types import ( + ColumnMetadataExpansionTestCase, HeaderMetadataParityTestCase, ModelConfigTierTestCase, ) from tests.integration.src.sqlbuild.compiler.model_config.helpers import ( HeaderMetadataParity, + column_metadata_outcome, generated_header_metadata, header_metadata_parity, model_config_engine_outcome, @@ -28,22 +30,65 @@ count=4000, expected_minimum_parsed=400, expected_minimum_rejected=2000, + expected_maximum_later_validated=60, ) ], ids=lambda case: case.description, ) -def test_given_generated_header_metadata_when_parsing_then_native_matches_yaml_schema_parsers( - test_case: HeaderMetadataParityTestCase, +def test_given_generated_header_metadata_when_attaching_then_model_loop_matches_yaml_parsers( + test_case: HeaderMetadataParityTestCase, tmp_path: Path ) -> None: parity: HeaderMetadataParity = header_metadata_parity( - headers=generated_header_metadata(rng=random.Random(test_case.seed), count=test_case.count) + project_dir=tmp_path, + headers=generated_header_metadata(rng=random.Random(test_case.seed), count=test_case.count), ) assert ( parity.mismatches, parity.parsed >= test_case.expected_minimum_parsed, parity.rejected >= test_case.expected_minimum_rejected, - ) == ([], True, True), (parity.parsed, parity.rejected) + parity.later_validated <= test_case.expected_maximum_later_validated, + ) == ([], True, True, True), (parity.parsed, parity.rejected, parity.later_validated) + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnMetadataExpansionTestCase( + description="a project variable template expands in a column type", + header_column="amount (type '${column_type}')", + expected_type="BIGINT", + expected_error=None, + ), + ColumnMetadataExpansionTestCase( + description="an unknown template variable in a column type is rejected", + header_column="amount (type '${target.schema}')", + expected_type=None, + expected_error="model config references unknown variable 'target.schema'", + ), + ColumnMetadataExpansionTestCase( + description="a macro call in a column description is rejected", + header_column="amount (description '@cents(amount)')", + expected_type=None, + expected_error="model config field 'columns.amount.description' does not allow macros", + ), + ColumnMetadataExpansionTestCase( + description="a macro marker after a space in a column type is rejected", + header_column="amount (type '@cents\u2003(amount)')", + expected_type=None, + expected_error="model config field 'columns.amount.type' does not allow macros", + ), + ], + ids=lambda case: case.description, +) +def test_given_template_or_macro_column_metadata_when_attaching_then_loop_expands_or_rejects( + test_case: ColumnMetadataExpansionTestCase, tmp_path: Path +) -> None: + column_type, error = column_metadata_outcome( + project_dir=tmp_path, header_column=test_case.header_column + ) + + assert (column_type, error) == (test_case.expected_type, test_case.expected_error) @pytest.mark.parametrize( @@ -53,16 +98,14 @@ def test_given_generated_header_metadata_when_parsing_then_native_matches_yaml_s description="shipped native stages", engine="native", expected_native_calls={ - "parse_model_header_metadata": 2, - "expand_config_templates": 1, + "expand_config_templates": 0, }, ), ModelConfigTierTestCase( description="native preview", engine="native-preview", expected_native_calls={ - "parse_model_header_metadata": 2, - "expand_config_templates": 1, + "expand_config_templates": 0, }, ), ], diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index 5a13a6279a..d03438f4fd 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -83,14 +83,6 @@ class DiagnosticUxCase: expected_partial: bool = True -@dataclass(frozen=True) -class ColumnSuggestionCase: - description: str - name: str - columns: tuple[str, ...] - expected_match: str | None - - @dataclass(frozen=True) class SemanticTriageCase: description: str @@ -386,6 +378,16 @@ class FunctionArgumentTypeCase: expected_codes: tuple[str, ...] = () +@dataclass(frozen=True) +class StarShapeOrderTestCase: + """A case-sensitive star model discovered before the upstream model it expands.""" + + description: str + files: dict[str, str] + star_model: str + expected_shape: dict[str, str] + + type CompileOutcome = tuple[int, dict[str, object], dict[str, bytes]] type PreparedCompile = tuple[ CompileOutcome, tuple[str, ...], dict[str, object | None], tuple[bytes, ...] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 5dad53d70c..141240c7bf 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -3,21 +3,21 @@ import json from collections.abc import Callable from pathlib import Path -from typing import Any +from typing import Any, NamedTuple import pytest import sqlbuild._native as native_module from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, - CompileAuditInput, CompiledLineageColumnFact, - CompiledModel, - CompiledProject, ) +from sqlbuild.compiler.compile.types import AuditInput, CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.manifest.main.build import build_manifest @@ -57,7 +57,7 @@ ) -def compile_audits_for_project(*, project_dir: Path) -> tuple[CompileAuditInput, ...]: +def compile_audits_for_project(*, project_dir: Path) -> tuple[AuditInput, ...]: """Compile one project's audit inputs with the real DuckDB compiler path.""" return build_compile_inputs( @@ -67,9 +67,7 @@ def compile_audits_for_project(*, project_dir: Path) -> tuple[CompileAuditInput, ).audit_inputs -def audit_input_projection( - *, audits: tuple[CompileAuditInput, ...] -) -> tuple[tuple[object, ...], ...]: +def audit_input_projection(*, audits: tuple[AuditInput, ...]) -> tuple[tuple[object, ...], ...]: """Project attachment-relevant fields for direct/generated equivalence.""" return tuple( @@ -158,7 +156,7 @@ def write_semantic_binding_project( def compile_reshaped_star_model( *, project_dir: Path, query_sql: str -) -> tuple[CompiledProject, CompiledModel]: +) -> tuple[CompiledProjectView, CompiledModelView]: """Compile one reshaped-star model over the shared upstream order models.""" (project_dir / "sqlbuild_project.toml").write_text( @@ -174,11 +172,11 @@ def compile_reshaped_star_model( f"MODEL (description 'Test model reshaped_orders.', materialized table);\n{query_sql}", encoding="utf-8", ) - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), ) - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} return project, models_by_name["reshaped_orders"] @@ -221,3 +219,50 @@ def traced(catalog: object, request: tuple[Any, ...], *rules: Any) -> Any: monkeypatch.setattr(native_module, "infer_expression_source_shapes", traced) return batches + + +def count_declaration_resolutions(*, monkeypatch: pytest.MonkeyPatch) -> list[int]: + """Count the native model loop's declaration projections.""" + + counts: list[int] = [0] + native_loop_type: type = native_module.NativeModelLoop + monkeypatch.setattr( + native_module, + "NativeModelLoop", + lambda *arguments: _ResolutionCountingLoop(native_loop_type(*arguments), counts), + ) + return counts + + +class _ResolutionCountingLoop(NamedTuple): + native_loop: Any + counts: list[int] + + def run(self, *arguments: object) -> tuple[list[object], tuple[int, int, int, int, int]]: + inputs, counts = self.native_loop.run(*arguments) + self.counts[0] += counts[2] + return inputs, counts + + +_OFFLINE_POSTGRES_CONNECTION: dict[str, object] = { + "host": "localhost", + "database": "warehouse", + "schema": "analytics", +} + + +def postgres_semantic_shapes( + *, project_dir: Path, files: dict[str, str] +) -> dict[str, dict[str, str]]: + """Compile `files` offline as a Postgres project and return every published model shape.""" + + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + project: CompiledProjectView = compile_project( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=PostgresAdapter(), + resolved_connection=_OFFLINE_POSTGRES_CONNECTION, + ) + return semantic_shapes(project=project) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py index 103fd56042..37d65b5af1 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompileAuditInput +from sqlbuild.compiler.compile.types import AuditInput from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( AuditFactoryCompileIntegrationTestCase, ) @@ -36,6 +36,7 @@ def test_given_generated_and_direct_attachments_when_compiled_then_audit_inputs_ SELECT * FROM __ref("@model") WHERE NOT (@expression) """, "python/factories/quality.py": """ +from sqlbuild.compiler.compile.types import AuditInput from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory @audit_factory @@ -51,7 +52,7 @@ def order_quality(): """, }, ) - generated: tuple[CompileAuditInput, ...] = compile_audits_for_project(project_dir=tmp_path) + generated: tuple[AuditInput, ...] = compile_audits_for_project(project_dir=tmp_path) write_repo_files( tmp_path, @@ -67,7 +68,7 @@ def order_quality(): """, }, ) - direct: tuple[CompileAuditInput, ...] = compile_audits_for_project(project_dir=tmp_path) + direct: tuple[AuditInput, ...] = compile_audits_for_project(project_dir=tmp_path) assert len(generated) == test_case.expected_audit_count assert audit_input_projection(audits=generated) == audit_input_projection(audits=direct) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py index 0719d710ad..3ae7073678 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_diff_selectors.py @@ -6,7 +6,7 @@ import pytest from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline._helpers.diff import ( @@ -102,7 +102,7 @@ def test_given_compiled_project_when_resolving_diff_selectors_then_returns_expec ) -> None: write_repo_files(tmp_path, _PROJECT_FILES) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProject = compile_project_for_diff_environment( + project: CompiledProjectView = compile_project_for_diff_environment( discovered_inputs=discovered_inputs, adapter=DuckDbAdapter(), target_name="dev", diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py index 6605544e64..b14f5b45d7 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py @@ -1,7 +1,6 @@ """Expression schemas are memoized only within their owning compiler catalog.""" import json -from dataclasses import replace from pathlib import Path from typing import Any @@ -9,7 +8,8 @@ from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from tests.integration.src.sqlbuild.compiler.lineage.helpers import compiled_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( ExpressionBatchCase, @@ -118,7 +118,7 @@ def test_given_restored_project_without_catalog_when_reading_shapes_then_infers_ f" - name: {name}\n description: Test source {name}.\n expression: {expression}\n" for name, expression in test_case.expressions.items() ) - project: CompiledProject = compiled_project( + project: CompiledProjectView = compiled_project( project_dir=tmp_path, files={ "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', @@ -127,7 +127,7 @@ def test_given_restored_project_without_catalog_when_reading_shapes_then_infers_ ) restored: dict[str, dict[str, str]] = semantic_shapes( - project=replace(project, binding_catalog=None) + project=project_with(project=project, binding_catalog=None) ) assert {name: restored.get(name) for name in test_case.expected_shapes} == ( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py index 91c6d5289c..c53e80cd4f 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_declaration_context.py @@ -9,7 +9,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project import compile_project @@ -72,7 +72,7 @@ def test_given_active_adapter_when_macro_renders_constant_then_uses_adapter_sql( adapter: BaseAdapter = test_case.adapter_factory() discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discovered_inputs, adapter=adapter, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py index 45ab004033..1b95960f7c 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_macro_load_count.py @@ -3,12 +3,10 @@ from collections.abc import Callable from pathlib import Path from typing import cast -from unittest.mock import Mock import pytest from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.scopes.models import DeclarationRecord from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -17,6 +15,7 @@ ) from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( build_manifest_for_pipeline_result, + count_declaration_resolutions, run_compile_pipeline_for_project, ) @@ -59,8 +58,7 @@ def test_given_side_effect_macro_when_compiling_manifest_then_imports_macros_onc write_repo_files: Callable[[Path, dict[str, str]], None], monkeypatch: pytest.MonkeyPatch, ) -> None: - resolver: Mock = Mock(wraps=attachment_core.resolve_declaration_context) - monkeypatch.setattr(attachment_core, "resolve_declaration_context", resolver) + resolutions: list[int] = count_declaration_resolutions(monkeypatch=monkeypatch) write_repo_files(tmp_path, test_case.project_files) result: CompilePipelineResult = run_compile_pipeline_for_project( @@ -75,7 +73,7 @@ def test_given_side_effect_macro_when_compiling_manifest_then_imports_macros_onc import_log: str = (tmp_path / "macro_import_log.txt").read_text(encoding="utf-8") assert import_log.count("import") == test_case.expected_macro_import_count - assert resolver.call_count == test_case.expected_declaration_resolution_count + assert resolutions[0] == test_case.expected_declaration_resolution_count @pytest.mark.parametrize( @@ -106,8 +104,7 @@ def test_given_composed_macro_when_compiling_then_scope_and_manifest_emit_depend write_repo_files: Callable[[Path, dict[str, str]], None], monkeypatch: pytest.MonkeyPatch, ) -> None: - resolver: Mock = Mock(wraps=attachment_core.resolve_declaration_context) - monkeypatch.setattr(attachment_core, "resolve_declaration_context", resolver) + resolutions: list[int] = count_declaration_resolutions(monkeypatch=monkeypatch) write_repo_files(tmp_path, test_case.project_files) result: CompilePipelineResult = run_compile_pipeline_for_project( @@ -136,4 +133,4 @@ def test_given_composed_macro_when_compiling_then_scope_and_manifest_emit_depend assert macro_node["depends_on"] == { "macros": [f"macro.demo.{name}" for name in test_case.expected_dependencies] } - assert resolver.call_count == test_case.expected_declaration_resolution_count + assert resolutions[0] == test_case.expected_declaration_resolution_count diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 505837d6d5..425be5f070 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -10,7 +10,8 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target -from sqlbuild.compiler.compile.models import CompiledProject, InferredColumn +from sqlbuild.compiler.compile.models import InferredColumn +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.lineage.types import InferredNullability @@ -98,7 +99,7 @@ def test_given_typed_cte_chain_when_compiling_contract_then_type_remains_proven( write_repo_files(tmp_path, test_case.project_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discovered_inputs, adapter=DuckDbAdapter(), ) @@ -609,7 +610,7 @@ def test_given_snowflake_local_target_namespace_when_compiling_then_targets_reso write_repo_files(tmp_path, test_case.project_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discovered_inputs, adapter=SnowflakeAdapter(), no_sql_validation=True, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py index f9d5697d63..3006b453fd 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_measurement_audits.py @@ -7,7 +7,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.types import AuditEvaluationMode, ThresholdOperator -from sqlbuild.compiler.compile.models import CompiledAudit +from sqlbuild.compiler.compile.types import CompiledAuditView from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import AuditPlanEntry from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -88,7 +88,7 @@ def test_given_attached_measurement_audit_when_compiling_then_full_contract_reac adapter=DuckDbAdapter(), ) - compiled: CompiledAudit = result.project.audits[0] + compiled: CompiledAuditView = result.project.audits[0] planned: AuditPlanEntry = result.plan_output.audit_entries[0] assert compiled.evaluation_mode == AuditEvaluationMode.MEASUREMENT assert compiled.measurement_contract is not None @@ -255,7 +255,7 @@ def test_given_standalone_measurement_with_header_policy_when_compiling_then_it_ project_dir=tmp_path, adapter=DuckDbAdapter() ) - audit: CompiledAudit = result.project.audits[0] + audit: CompiledAuditView = result.project.audits[0] assert audit.thresholds is not None and audit.thresholds.warn is not None assert audit.thresholds.warn.operator == ThresholdOperator.OUTSIDE assert audit.minimum_samples == test_case.expected_minimum_samples diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py index 8116e86214..d13eb68ab8 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -9,9 +9,11 @@ from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile._helpers.assembly import project as assembly +from sqlbuild.compiler.compile._helpers.assembly import source_bindings +from sqlbuild.compiler.compile.classes import semantic_completion_inputs from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -115,7 +117,7 @@ def test_given_catalog_when_rebinding_physical_shapes_then_does_not_reuse_stale_ "MODEL (description 'Test model report.', database warehouse, schema analytics);\n" + test_case.sql ) - project: CompiledProject = build_compiled_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=SnowflakeAdapter() ) assert project.binding_catalog is not None @@ -177,7 +179,10 @@ def test_given_sql_when_case_policy_enabled_then_fused_validation_is_reused( def repeated_binding(**kwargs: Any) -> None: raise AssertionError("already validated SQL was normalized and bound again") - monkeypatch.setattr(assembly, "get_complete_schema_binding_request", repeated_binding) + monkeypatch.setattr(source_bindings, "get_complete_schema_binding_request", repeated_binding) + monkeypatch.setattr( + semantic_completion_inputs, "get_complete_schema_binding_request", repeated_binding + ) assert ( main( [ diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py index f54b291d9a..4b1ebb1190 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py @@ -10,10 +10,8 @@ from _pytest.capture import CaptureResult from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile._helpers.diagnostics.details import closest_column from sqlbuild.compiler.compile._helpers.diagnostics.help import semantic_help_catalogue from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( - ColumnSuggestionCase, DiagnosticUxCase, ) @@ -210,24 +208,5 @@ def test_given_semantic_codes_when_explaining_then_each_code_owns_distinct_help( ) -@pytest.mark.parametrize( - "test_case", - [ - ColumnSuggestionCase("abbreviation", "qty", ("quantity", "id", "status"), "quantity"), - ColumnSuggestionCase("typo", "quantitty", ("quantity", "status"), "quantity"), - ColumnSuggestionCase("no close match", "unrelated_field", ("quantity", "id"), None), - ColumnSuggestionCase("unicode case folding", "straße", ("STRASSE",), "STRASSE"), - ], - ids=lambda case: case.description, -) -def test_given_column_spelling_when_suggesting_then_uses_bounded_edit_distance( - test_case: ColumnSuggestionCase, -) -> None: - assert ( - closest_column(name=test_case.name, columns=dict.fromkeys(test_case.columns, "INTEGER")) - == test_case.expected_match - ) - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py index ff4cb8db7c..dd1bb0e179 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py @@ -8,7 +8,7 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project @@ -139,7 +139,7 @@ def test_given_snowflake_query_when_inferring_rich_types_then_preserves_proven_o "MODEL (description 'Test model report.', materialized view, database warehouse, schema analytics);\n" + test_case.sql ) - project: CompiledProject = build_compiled_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=SnowflakeAdapter(), column_lineage_mode=ColumnLineageMode.RICH, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py index e0be6124d6..a69974eb43 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -12,7 +12,8 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics -from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -94,7 +95,7 @@ def test_given_effective_target_when_binding_quotes_then_honours_session_setting main(["--project-dir", str(tmp_path), "compile", "--target", "selected", "--json"]) result: dict[str, Any] = json.loads(capsys.readouterr().out) assert tuple(item["code"] for item in result["diagnostics"]) == test_case.expected_codes - project: CompiledProject = build_compiled_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=SnowflakeAdapter(), selected_target="selected", diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py index c794065571..b0641ccc5f 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py @@ -12,9 +12,8 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, - CompiledModel, - CompiledProject, ) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( @@ -61,7 +60,7 @@ def test_given_mixed_branch_types_when_compiling_then_known_types_match_duckdb( encoding="utf-8", ) - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), ) @@ -113,11 +112,11 @@ def test_given_union_type_recovery_when_compiling_then_cte_lineage_is_preserved( encoding="utf-8", ) - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), ) - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} lineage: Sequence[CompiledLineageColumnFact] | None = models_by_name[ "combined_orders" ].fast_lineage_columns diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py index 3d3b9ece48..a7d2d4464d 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_sql_test_input_extraction.py @@ -9,8 +9,8 @@ from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, - CompiledProject, ) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ExpectedCountTestCase @@ -53,7 +53,7 @@ def test_given_expanded_mixed_mode_tests_when_compiling_project_then_complete_in }, ) - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=DuckDbAdapter(), ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_star_shape_order.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_star_shape_order.py new file mode 100644 index 0000000000..17b394313f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_star_shape_order.py @@ -0,0 +1,40 @@ +"""Semantic shapes publish a star model only after the upstream shapes it expands.""" + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import StarShapeOrderTestCase +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import postgres_semantic_shapes + + +@pytest.mark.parametrize( + "test_case", + [ + StarShapeOrderTestCase( + description="quoted upstream column keeps its quoting through a star listed first", + files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "postgres"\n', + "models/a_orders_wide.sql": ( + "MODEL (description 'Every order column.', schema analytics);\n" + 'SELECT * FROM __ref("z_orders")\n' + ), + "models/z_orders.sql": ( + "MODEL (description 'Orders.', schema analytics);\n" + "SELECT 1 AS \"OrderId\", 'placed' AS status\n" + ), + }, + star_model="a_orders_wide", + expected_shape={"status": "UNKNOWN", '"OrderId"': "UNKNOWN"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_star_model_before_its_upstream_when_shaping_then_upstream_shape_is_used( + test_case: StarShapeOrderTestCase, tmp_path: Path +) -> None: + shapes: dict[str, dict[str, str]] = postgres_semantic_shapes( + project_dir=tmp_path, files=test_case.files + ) + + assert shapes[test_case.star_model] == test_case.expected_shape diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index aed0a19048..14ced6d8fc 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -18,7 +18,15 @@ CompileModelConfig, CompileSqlReference, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompiledSeedView, + CompiledSourceView, +) +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredSchemaFile, DiscoveredSeedFile, @@ -88,10 +96,10 @@ def build_project_with_targets( seed_locations: dict[str, str | None] | None = None, incremental_models: tuple[_IncrementalModelSpec, ...] = (), source_names: tuple[tuple[str, str, str], ...] = (), -) -> CompiledProject: +) -> CompiledProjectView: """Build a minimal CompiledProject with explicit target schemas.""" - models: list[CompiledModel] = [] + models: list[CompiledModelView] = [] model_name: str target_schema: str | None for model_name, target_schema in (model_locations or {}).items(): @@ -152,7 +160,7 @@ def build_project_with_targets( ) ) - seeds: list[CompiledSeed] = [] + seeds: list[CompiledSeedView] = [] seed_name: str for seed_name, target_schema in (seed_locations or {}).items(): seeds.append( @@ -181,7 +189,7 @@ def build_project_with_targets( ) ) - sources: list[CompiledSource] = [] + sources: list[CompiledSourceView] = [] source_spec: tuple[str, str, str] for source_spec in source_names: sources.append( @@ -215,23 +223,23 @@ def build_project_with_targets( def with_leading_enforced_model_contracts( - project: CompiledProject, *, columns_by_model: dict[str, tuple[str, ...]] -) -> CompiledProject: + project: CompiledProjectView, *, columns_by_model: dict[str, tuple[str, ...]] +) -> CompiledProjectView: """Attach enforced contracts to leading models in a compiled test project.""" - models: list[CompiledModel] = list(project.models) + models: list[CompiledModelView] = list(project.models) for index, (model_name, column_names) in enumerate(columns_by_model.items()): - model: CompiledModel = models[index] + model: CompiledModelView = models[index] assert model.name == model_name models[index] = replace( - model, + compiled_model_record(model), config=replace(model.config, values=model.config.values | {"contract": "enforced"}), schema_entry=SchemaModelEntry( name=model.name, columns=tuple(SchemaColumn(name=name) for name in column_names), ), ) - return replace(project, models=tuple(models)) + return project_with(project=project, models=tuple(models)) def build_deferred_locations_from_map( diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index f8c84b04ed..f530674685 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -15,7 +15,7 @@ CompileModelConfig, CompileSqlReference, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_sql from sqlbuild.compiler.planner.models import ( BackfillResult, @@ -35,7 +35,7 @@ def build_model( query_sql: str, config: dict[str, object], ref_names: tuple[str, ...], -) -> CompiledModel: +) -> CompiledModelView: """Build a minimal CompiledModel for resolve integration tests.""" raw_schema: object | None = config.get("schema") @@ -72,7 +72,7 @@ class _ResolveResult: def resolve_and_execute( *, - model: CompiledModel, + model: CompiledModelView, snapshot: WarehouseSnapshot, model_locations: dict[str, CompiledRelationLocation], source_map: dict[str, SourceEntry], @@ -114,7 +114,7 @@ def resolve_and_execute( return _ResolveResult(resolved_sql=resolved_sql, rows=rows, column_types=column_types) -def build_empty_input_model(*, cursor_start: str | None) -> CompiledModel: +def build_empty_input_model(*, cursor_start: str | None) -> CompiledModelView: """Build a delete_insert cursor model reading the raw_events source.""" config: dict[str, object] = { diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py index bf5731aa2a..8c6dab8361 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/test_resolve.py @@ -8,9 +8,9 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledRelationLocation, ) +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ModelCursorSnapshot, WarehouseSnapshot from sqlbuild.compiler.planner.types import CursorType @@ -76,7 +76,7 @@ def test_given_refs_when_resolving_and_executing_then_returns_expected_rows( for statement in test_case.setup_sql: connection.execute(statement) - model: CompiledModel = build_model( + model: CompiledModelView = build_model( name="test_model", query_sql=test_case.query_sql, config=test_case.model_config, @@ -143,7 +143,7 @@ def test_given_incremental_source_when_resolving_and_executing_then_returns_curs for statement in test_case.setup_sql: connection.execute(statement) - model: CompiledModel = build_model( + model: CompiledModelView = build_model( name="fact_events", query_sql=test_case.query_sql, config=test_case.model_config, @@ -236,7 +236,7 @@ def test_given_fully_bounded_overrides_when_snapshot_missing_then_sql_uses_exact "INSERT INTO staging.raw_events " f"SELECT row_number() OVER (), value FROM (VALUES {test_case.inserted_values}) AS v(value)" ) - model: CompiledModel = build_model( + model: CompiledModelView = build_model( name="fact_events", query_sql=( "SELECT event_id, cursor_value, __cursor_start() AS range_start, " @@ -321,7 +321,7 @@ def test_given_view_filter_and_physical_watermark_when_resolving_then_filters_on statement: str for statement in test_case.setup_sql: connection.execute(statement) - model: CompiledModel = build_model( + model: CompiledModelView = build_model( name="fact_events", query_sql=test_case.query_sql, config=test_case.model_config, @@ -377,7 +377,7 @@ def test_given_unavailable_planning_watermark_when_resolving_then_fails_closed( test_case: ResolveWatermarkFailureTestCase, connection: Any, ) -> None: - model: CompiledModel = build_model( + model: CompiledModelView = build_model( name="fact_events", query_sql='SELECT event_id, event_time FROM __source("raw_events")', config={ @@ -583,7 +583,7 @@ def test_given_sources_when_resolving_and_executing_then_returns_expected_rows_a for statement in test_case.setup_sql: connection.execute(statement) - model: CompiledModel = build_model( + model: CompiledModelView = build_model( name="test_model", query_sql=test_case.query_sql, config=test_case.model_config, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index a99efc65fe..ef7dab0c6b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -20,7 +20,11 @@ CompileModelConfig, CompileSqlTestCte, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.discovery.models import ( DiscoveredSqlTestBlock, DiscoveredSqlTestFile, @@ -46,7 +50,7 @@ def build_test_and_project( test_case: ExecuteChainTestCase, -) -> tuple[CompiledSqlTest, CompiledProject]: +) -> tuple[CompiledSqlTest, CompiledProjectView]: """Build a CompiledSqlTest and CompiledProject from a test case.""" authored_ctes: list[CompileSqlTestCte] = [] @@ -101,7 +105,7 @@ def build_test_and_project( ), ) - models: list[CompiledModel] = [] + models: list[CompiledModelView] = [] model_name: str query_sql: str for model_name, query_sql in test_case.model_queries.items(): @@ -138,7 +142,7 @@ def build_test_and_project( ) ) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="test_run", effective_target_name=None, effective_connection={}, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index dfbfa47666..ea88190766 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -14,7 +14,7 @@ CompiledSqlTest, CompileSqlTestCte, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project @@ -79,7 +79,7 @@ def test_given_parameterized_model_test_when_compiled_and_executed_then_duckdb_p + "\n", }, ) - project: CompiledProject = build_compiled_project( + project: CompiledProjectView = build_compiled_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter, ) @@ -310,7 +310,7 @@ def test_given_chain_when_executing_resolved_sql_then_produces_expected_rows( connection: Any, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) entry: SqlTestPlanEntry @@ -478,7 +478,7 @@ def test_given_sql_analysis_enabled_when_planning_chain_then_step_sql_remains_ex connection: Any, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) entry: SqlTestPlanEntry diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index e1949b84f8..e02a9b154d 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -11,7 +11,7 @@ CompiledObjectKey, CompiledRelationLocation, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType from sqlbuild.compiler.discovery.models import ( DiscoveredAuditBlock, DiscoveredAuditFile, @@ -91,7 +91,7 @@ def test_given_audit_when_executing_resolved_sql_then_returns_expected_rows( for sql in test_case.setup_sql: connection.execute(sql) - audit: CompiledAudit = CompiledAudit( + audit: CompiledAuditView = CompiledAudit( key=CompiledObjectKey( resource_type=CompiledResourceType.AUDIT, name="test_audit", diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 57f33b72c8..eb63b583cc 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -6,7 +6,7 @@ import pytest from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.main.project import compile_project @@ -66,7 +66,7 @@ def test_given_partial_fixture_when_compiling_and_planning_then_completes_requir discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discovered_inputs, adapter=adapter, ) @@ -135,7 +135,7 @@ def test_given_untyped_null_fixture_when_planning_then_applies_authoritative_typ discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discovered_inputs, adapter=adapter, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index 0984716805..c658ac2fc1 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -9,12 +9,16 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledRelationLocation, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.warehouse.snapshot import gather_warehouse_snapshot @@ -244,7 +248,7 @@ def test_given_warehouse_state_when_gathering_snapshot_then_returns_expected( render_framework_type=adapter.render_framework_type, ) - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( model_locations=test_case.model_locations, model_deps=test_case.model_deps, seed_locations=test_case.seed_locations, @@ -288,7 +292,7 @@ def test_given_no_target_schemas_when_gathering_snapshot_then_returns_empty( connection: Any, execute: Any, ) -> None: - project: CompiledProject = build_project_with_targets(model_locations={}, seed_locations={}) + project: CompiledProjectView = build_project_with_targets(model_locations={}, seed_locations={}) snapshot: WarehouseSnapshot = gather_warehouse_snapshot( project=project, adapter=adapter, @@ -326,7 +330,7 @@ def test_given_sources_when_gathering_source_columns_then_filters_metadata_names for statement in test_case.setup_sql: connection.execute(statement) adapter: RecordingDuckDbAdapter = RecordingDuckDbAdapter() - project: CompiledProject = build_project_with_targets(source_names=test_case.source_names) + project: CompiledProjectView = build_project_with_targets(source_names=test_case.source_names) columns: dict[str, tuple[ColumnInfo, ...]] = gather_source_columns( project=project, @@ -470,7 +474,7 @@ def test_given_incremental_models_when_gathering_cursor_snapshots_then_returns_e def _track_progress(message: str) -> None: progress_calls.append(message) - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( model_locations={"raw_orders": "staging"}, incremental_models=( _IncrementalModelSpec( @@ -568,7 +572,7 @@ def _track_progress(message: str) -> None: deferred: dict[str, CompiledRelationLocation] | None = test_case.deferred_locations - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( model_locations={"raw_orders": "staging"}, incremental_models=(_INCREMENTAL_MODEL,), ) @@ -622,7 +626,7 @@ def _record_execute(*, connection: Any, sql: str) -> Any: statements.append(sql) return connection.execute(sql) - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( model_locations={"raw_events": "staging"}, incremental_models=( _IncrementalModelSpec( @@ -683,7 +687,7 @@ def test_given_timezone_cursor_text_when_gathering_snapshot_then_normalizes_ware "INSERT INTO staging.raw_events VALUES (?)", [(value,) for value in test_case.warehouse_values], ) - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( model_locations={"raw_events": "staging"}, incremental_models=( _IncrementalModelSpec( @@ -732,7 +736,7 @@ def _fail_eligible_execute(*, connection: Any, sql: str) -> Any: statements.append(sql) return connection.execute(sql) - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( model_locations={"raw_events": "staging"}, incremental_models=( _IncrementalModelSpec( @@ -803,7 +807,7 @@ def _record_execute(*, connection: Any, sql: str) -> Any: statements.append(sql) return connection.execute(sql) - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( incremental_models=( _IncrementalModelSpec( name="daily_events", @@ -880,7 +884,7 @@ def _record_execute(*, connection: Any, sql: str) -> Any: statements.append(sql) return connection.execute(sql) - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( model_locations={"valid_upstream": "staging", "invalid_upstream": "staging"}, incremental_models=( _IncrementalModelSpec( @@ -954,7 +958,7 @@ def test_given_selected_invalid_cursor_model_when_gathering_snapshot_then_raises execute: Any, ) -> None: connection.execute("CREATE TABLE staging.invalid_upstream (id INTEGER)") - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( model_locations={"invalid_upstream": "staging"}, incremental_models=( _IncrementalModelSpec( @@ -1017,7 +1021,7 @@ def test_given_full_refresh_with_time_only_watermark_contract_when_gathering_the connection: Any, execute: Any, ) -> None: - project: CompiledProject = build_project_with_targets( + project: CompiledProjectView = build_project_with_targets( model_locations={"upstream_events": "staging"}, incremental_models=( _IncrementalModelSpec( @@ -1041,8 +1045,8 @@ def test_given_full_refresh_with_time_only_watermark_contract_when_gathering_the ), ), ) - upstream: CompiledModel = replace( - project.models[0], + upstream: CompiledModelView = replace( + compiled_model_record(project.models[0]), config=replace( project.models[0].config, values=project.models[0].config.values | {"contract": "enforced"}, @@ -1052,7 +1056,7 @@ def test_given_full_refresh_with_time_only_watermark_contract_when_gathering_the columns=(SchemaColumn(name="event_time", type=test_case.declared_type),), ), ) - project = replace(project, models=(upstream, project.models[1])) + project = project_with(project=project, models=(upstream, project.models[1])) daily_key: CompiledObjectKey = project.models[1].key with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 7c7fde33f7..94fec55212 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -26,7 +26,17 @@ CompileSqlReference, FunctionArgument, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompiledSeedView, + CompiledSourceView, + FunctionLanguage, +) +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, @@ -73,10 +83,10 @@ def build_project_from_test_case( test_case: BuildExecutionPlanTestCase, -) -> CompiledProject: +) -> CompiledProjectView: """Build a CompiledProject from an integration test case.""" - models: list[CompiledModel] = [] + models: list[CompiledModelView] = [] model_name: str target_schema: str for model_name, target_schema in test_case.model_locations.items(): @@ -110,7 +120,7 @@ def build_project_from_test_case( ) ) - seeds: list[CompiledSeed] = [] + seeds: list[CompiledSeedView] = [] seed_name: str for seed_name, target_schema in test_case.seed_locations.items(): seed_file_path: Path = _ensure_test_seed_file(seed_name) @@ -143,7 +153,7 @@ def build_project_from_test_case( ) ) - functions: list[CompiledFunction] = [] + functions: list[CompiledFunctionView] = [] function_name: str for function_name, target_schema in test_case.function_locations.items(): body_sql: str = test_case.function_bodies.get(function_name, "value = 1") @@ -204,7 +214,7 @@ def _settings_bool(settings: dict[str, object], key: str, *, default: bool) -> b def build_project_from_source_cursor_input_test_case( test_case: SourceCursorInputPlanErrorTestCase, -) -> CompiledProject: +) -> CompiledProjectView: """Build a project with one incremental model reading one source.""" source_entry: SourceEntry = SourceEntry( @@ -212,7 +222,7 @@ def build_project_from_source_cursor_input_test_case( schema=test_case.source_schema, table=test_case.source_table, ) - source: CompiledSource = CompiledSource( + source: CompiledSourceView = CompiledSource( key=CompiledObjectKey( resource_type=CompiledResourceType.SOURCE, name=test_case.source_name, @@ -227,7 +237,7 @@ def build_project_from_source_cursor_input_test_case( source_entries=(source_entry,), ), ) - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey( resource_type=CompiledResourceType.MODEL, name=test_case.model_name, @@ -267,7 +277,7 @@ def build_project_from_source_cursor_input_test_case( ) -def build_future_cursor_project() -> CompiledProject: +def build_future_cursor_project() -> CompiledProjectView: """Build one timestamp incremental source project for future safety tests.""" case: SourceCursorInputPlanErrorTestCase = SourceCursorInputPlanErrorTestCase( @@ -281,13 +291,13 @@ def build_future_cursor_project() -> CompiledProject: cursor_input_column="occurred_at", expected_error_fragment="", ) - project: CompiledProject = build_project_from_source_cursor_input_test_case(case) - model: CompiledModel = project.models[0] - return replace( - project, + project: CompiledProjectView = build_project_from_source_cursor_input_test_case(case) + model: CompiledModelView = project.models[0] + return project_with( + project=project, models=( replace( - model, + compiled_model_record(model), config=replace( model.config, values={ @@ -314,13 +324,13 @@ def write_previous_function_fingerprints( test_case, function_bodies={**test_case.function_bodies, **test_case.previous_function_bodies}, ) - previous_project: CompiledProject = build_project_from_test_case(previous_case) - function_map: dict[str, CompiledFunction] = { + previous_project: CompiledProjectView = build_project_from_test_case(previous_case) + function_map: dict[str, CompiledFunctionView] = { function.name: function for function in previous_project.functions } function_name: str for function_name in test_case.previous_function_bodies: - function: CompiledFunction = function_map[function_name] + function: CompiledFunctionView = function_map[function_name] fingerprint_sql: str = build_compiled_function_fingerprint_sql(function) write_fingerprint( connection=connection, @@ -346,10 +356,10 @@ def write_previous_function_fingerprints( def build_project_from_format_test_case( test_case: FormatPlanIntegrationTestCase, -) -> CompiledProject: +) -> CompiledProjectView: """Build a CompiledProject from a format integration test case.""" - models: list[CompiledModel] = [] + models: list[CompiledModelView] = [] model_name: str target_schema: str for model_name, target_schema in test_case.model_locations.items(): @@ -379,7 +389,7 @@ def build_project_from_format_test_case( ) ) - seeds: list[CompiledSeed] = [] + seeds: list[CompiledSeedView] = [] seed_name: str for seed_name, target_schema in test_case.seed_locations.items(): seed_file_path: Path = _ensure_test_seed_file(seed_name) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index 8e38febc00..c59d3bccce 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -10,7 +10,9 @@ from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.output.main.plan import format_plan -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner._helpers.changes import detect as change_detection from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning from sqlbuild.compiler.planner.exceptions import PlannerInputError @@ -98,7 +100,7 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d monkeypatch: pytest.MonkeyPatch, ) -> None: connection.execute("CREATE TABLE staging.stg_orders AS SELECT 1 AS order_id") - project: CompiledProject = build_project_from_test_case( + project: CompiledProjectView = build_project_from_test_case( BuildExecutionPlanTestCase( description=test_case.description, setup_sql=(), @@ -118,8 +120,8 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d expected_reason={}, ) ) - project = replace( - project, + project = project_with( + project=project, settings=replace( project.settings, query_change_tracking=test_case.query_change_tracking, @@ -507,7 +509,7 @@ def test_given_view_and_table_with_target_default_when_assembling_plan_then_only ) -> None: connection.execute("CREATE OR REPLACE VIEW staging.orders_view AS SELECT 1 AS id") connection.execute("CREATE OR REPLACE TABLE staging.orders AS SELECT 1 AS id") - project: CompiledProject = build_project_from_test_case( + project: CompiledProjectView = build_project_from_test_case( BuildExecutionPlanTestCase( description=test_case.description, setup_sql=(), @@ -523,13 +525,13 @@ def test_given_view_and_table_with_target_default_when_assembling_plan_then_only ) ) view_model, table_model = project.models - project = replace( - project, + project = project_with( + project=project, effective_target_name="test", models=( view_model, replace( - table_model, + compiled_model_record(table_model), config=replace( table_model.config, table_type=ResolvedTableType( @@ -972,7 +974,7 @@ def test_given_future_upstream_when_building_real_plan_then_cap_has_structured_e adapter: DuckDbAdapter, connection: Any, ) -> None: - project: CompiledProject = build_future_cursor_project() + project: CompiledProjectView = build_future_cursor_project() connection.execute("CREATE SCHEMA IF NOT EXISTS raw") connection.execute("CREATE SCHEMA IF NOT EXISTS staging") connection.execute(f"CREATE TABLE raw.events (occurred_at {test_case.warehouse_type})") @@ -1016,7 +1018,7 @@ def test_given_future_upstream_and_error_policy_when_building_real_plan_then_fai adapter: DuckDbAdapter, connection: Any, ) -> None: - project: CompiledProject = build_future_cursor_project() + project: CompiledProjectView = build_future_cursor_project() connection.execute("CREATE SCHEMA IF NOT EXISTS raw") connection.execute("CREATE SCHEMA IF NOT EXISTS staging") connection.execute("CREATE TABLE raw.events (occurred_at TIMESTAMP)") diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index b61931c5ed..f532a31915 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -7,7 +7,6 @@ import random from collections import Counter from collections.abc import Callable -from dataclasses import replace from pathlib import Path from typing import Any @@ -21,9 +20,9 @@ from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileAdapterContext, - CompiledProject, - CompileProjectInputs, ) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView +from sqlbuild.compiler.compiled_project.main.inputs_with import inputs_with from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.types import CompilerEngine @@ -143,7 +142,7 @@ def seed_yml(schema: str) -> str: ) -def project_inputs(*, project_dir: Path, files: dict[str, str]) -> CompileProjectInputs: +def project_inputs(*, project_dir: Path, files: dict[str, str]) -> CompileInputsView: """Write `files` and attach their compile inputs as the compile pipeline does.""" _ = [ @@ -159,8 +158,8 @@ def project_inputs(*, project_dir: Path, files: dict[str, str]) -> CompileProjec def assemble_with( - *, inputs: CompileProjectInputs, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch -) -> CompiledProject: + *, inputs: CompileInputsView, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch +) -> CompiledProjectView: """Assemble `inputs` with `engine`.""" monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) @@ -168,12 +167,12 @@ def assemble_with( def recorded_assembly( - *, inputs: CompileProjectInputs, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch + *, inputs: CompileInputsView, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch ) -> tuple[object, tuple[tuple[str, ...], bool]]: """Assemble `inputs` with `engine`; return its assembly view and the input reads recorded.""" with COMPILE_INPUT_READS.recording() as reads: - project: CompiledProject = assemble_with( + project: CompiledProjectView = assemble_with( inputs=inputs, engine=engine, monkeypatch=monkeypatch ) return assembly_view(project), (reads.environment_names, reads.read_run_id) @@ -189,9 +188,9 @@ def assembly_outcome( """Attach and assemble `files`: `("ok", view)`, or the error's class, message, help and code.""" try: - inputs: CompileProjectInputs = project_inputs(project_dir=project_dir, files=files) - merged: CompileProjectInputs = replace( - inputs, effective_vars={**inputs.effective_vars, **(effective_vars or {})} + inputs: CompileInputsView = project_inputs(project_dir=project_dir, files=files) + merged: CompileInputsView = inputs_with( + inputs=inputs, effective_vars={**inputs.effective_vars, **(effective_vars or {})} ) return ( "ok", @@ -208,7 +207,7 @@ def assembly_outcome( ) -def assembly_view(project: CompiledProject) -> object: +def assembly_view(project: CompiledProjectView) -> object: """Return every resource fact native project assembly derives, with the diagnostics.""" return ( @@ -231,7 +230,7 @@ def rejected_opt_outs( """Attach `files` with `engine`; return whether each opt-out is rejected.""" monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) - inputs: CompileProjectInputs = project_inputs(project_dir=project_dir, files=files) + inputs: CompileInputsView = project_inputs(project_dir=project_dir, files=files) return { model_input.model_file.file_path.stem: model_input.rejected_sql_analysis_opt_out is not None for model_input in inputs.model_inputs diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index f7cf6d8ef6..f779634d5c 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -13,7 +13,7 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.types import CompilerEngine from sqlbuild.compiler.project_assembly.types import ProjectRequestRow @@ -85,7 +85,7 @@ def test_given_generated_projects_when_assembling_natively_then_matches_python_g calls: Counter[str] = python_resource_calls(monkeypatch=monkeypatch) rng: random.Random = random.Random(test_case.seed) project_dirs: list[Path] = [tmp_path / f"project_{index}" for index in range(test_case.count)] - inputs: list[CompileProjectInputs] = [ + inputs: list[CompileInputsView] = [ project_inputs( project_dir=project_dir, files=generated_assembly_files(rng=rng, model_count=test_case.model_count), @@ -184,7 +184,7 @@ def test_given_model_sql_the_parser_rejects_when_assembling_then_raises_python_e tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - inputs: CompileProjectInputs = project_inputs(project_dir=tmp_path, files=test_case.files) + inputs: CompileInputsView = project_inputs(project_dir=tmp_path, files=test_case.files) with pytest.raises(CompileInputError) as raised: _ = assemble_with(inputs=inputs, engine=CompilerEngine.NATIVE, monkeypatch=monkeypatch) @@ -208,7 +208,7 @@ def test_given_windows_environment_when_assembling_natively_then_env_lookups_mat test_case: WindowsEnvironmentTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(os, "environ", windows_environ(test_case.environment)) - inputs: CompileProjectInputs = project_inputs( + inputs: CompileInputsView = project_inputs( project_dir=tmp_path, files={ "sqlbuild_project.toml": _PROJECT_TOML, @@ -217,7 +217,7 @@ def test_given_windows_environment_when_assembling_natively_then_env_lookups_mat }, ) - native: CompiledProject = assemble_with( + native: CompiledProjectView = assemble_with( inputs=inputs, engine=CompilerEngine.NATIVE, monkeypatch=monkeypatch ) @@ -295,7 +295,7 @@ def test_given_project_when_assembling_natively_then_request_carries_graph_facts test_case: GraphFactsTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: requests: list[ProjectRequestRow] = record_assembly_requests(monkeypatch=monkeypatch) - project: CompiledProject = assemble_with( + project: CompiledProjectView = assemble_with( inputs=project_inputs(project_dir=tmp_path / "project", files=test_case.files), engine=CompilerEngine.NATIVE, monkeypatch=monkeypatch, @@ -344,7 +344,7 @@ def test_given_native_assembly_panics_when_assembling_then_raises_without_python test_case: NativePanicTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.NATIVE.value) - inputs: CompileProjectInputs = project_inputs( + inputs: CompileInputsView = project_inputs( project_dir=tmp_path / "project", files=test_case.files ) calls: Counter[str] = python_resource_calls(monkeypatch=monkeypatch) diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index 1759570dc4..a7fb008c2d 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -4,7 +4,7 @@ import random from collections import defaultdict -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass, fields, replace from itertools import chain from pathlib import Path @@ -12,12 +12,12 @@ import pytest +import sqlbuild._native as _native from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.attachment import declaration_scope from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros -from sqlbuild.compiler.compile._helpers.sql_tests.native import native_sql_test_ctes from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( DeclarationResolutionContext, @@ -36,10 +36,6 @@ from sqlbuild.compiler.frontier.types import CompilerEngine from sqlbuild.compiler.scopes.classes.native_scope_index import NativeScopeIndex from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path -from sqlbuild.compiler.scopes.main._declaration_visibility import declaration_visibility -from sqlbuild.compiler.scopes.main._native_expected_model_names import ( - native_expected_model_names, -) from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( resolve_scope_declaration_visibility, @@ -505,6 +501,13 @@ def _generated_expected_model_sql(*, rng: random.Random) -> str: return f"{rng.choice(_LEADING)}{rng.choice(_WITH)}{','.join(ctes)}{rng.choice(_TRAILING)}" +def _scan_outcome(outcome: tuple[str | None, Sequence[object]]) -> tuple[object, ...] | str: + """A scan's error, or its values as a tuple.""" + + error, values = outcome + return {True: lambda: tuple(values), False: lambda: cast(str, error)}[error is None]() + + def relationship_scans(*, sqls: list[str], syntax: SqlLexicalSyntax) -> RelationshipScans: """Scan each SQL text's test names, scenario names and test CTEs natively.""" @@ -512,9 +515,15 @@ def relationship_scans(*, sqls: list[str], syntax: SqlLexicalSyntax) -> Relation scenario_texts: list[tuple[str, str]] = [(sql, _SCENARIO_LABEL) for sql in sqls] outcomes: list[tuple[object, object, object]] = list( zip( - native_expected_model_names(texts=test_texts, scenario=False, syntax=syntax), - native_expected_model_names(texts=scenario_texts, scenario=True, syntax=syntax), - native_sql_test_ctes(texts=test_texts, syntax=syntax), + map( + _scan_outcome, + _native.scope_expected_model_names(test_texts, False, syntax.native_mapping), + ), + map( + _scan_outcome, + _native.scope_expected_model_names(scenario_texts, True, syntax.native_mapping), + ), + map(_scan_outcome, _native.scope_test_ctes(test_texts, syntax.native_mapping)), strict=True, ) ) @@ -778,6 +787,13 @@ def _resource_visibility( ) +_PATH_VISIBLE_REASONS: dict[ScopeKind, VisibilityReason] = { + ScopeKind.GLOBAL: VisibilityReason.GLOBAL, + ScopeKind.LOCAL: VisibilityReason.LOCAL_OWNER, + ScopeKind.INHERITED: VisibilityReason.INHERITED_ANCESTOR, +} + + def _path_visibility( *, resolver: DeclarationScopeResolver, resolution: DeclarationVisibility, target_path: Path ) -> _Visibility: @@ -799,8 +815,7 @@ def _path_visibility( ResourceKind.MODEL, f"" ) reasons: Iterator[tuple[DeclarationRecord, VisibilityReason | None]] = ( - (record, declaration_visibility(declaration=record, consumer=lexical_path)) - for record in visible + (record, _PATH_VISIBLE_REASONS[record.scope]) for record in visible ) return ( visible, diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 3807594f49..b75e604158 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -17,10 +17,13 @@ import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.analysis_session.types import DiagnosticRow from sqlbuild.compiler.compile._helpers.diagnostics.recovery import ( complete_semantic_diagnostics, ) -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.pipeline.main.graph import build_project_graph @@ -29,7 +32,6 @@ from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( complete_native_semantic_diagnostics, ) -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( generated_analysis_files, ) @@ -93,9 +95,9 @@ class SemanticInputs: """The arguments the Python compile passed to `complete_semantic_diagnostics`.""" - project: CompiledProject + project: CompiledProjectView profile: ExpressionInferenceProfile - binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]] + binding_results: dict[str, list[DiagnosticRow]] resource_sql_analysis: bool native_session: Any | None = None @@ -204,7 +206,7 @@ def captured_semantic_inputs( _ = path.write_text(contents, encoding="utf-8") captured: list[SemanticInputs] = [] - def capture(**arguments: Any) -> CompiledProject: + def capture(**arguments: Any) -> CompiledProjectView: captured.append(SemanticInputs(**arguments)) return complete_semantic_diagnostics(**arguments) @@ -223,18 +225,18 @@ def with_dialect(inputs: SemanticInputs, dialect: str | None) -> SemanticInputs: return replace( inputs, - project=replace(inputs.project, sql_analysis_dialect=dialect), + project=project_with(project=inputs.project, sql_analysis_dialect=dialect), profile=replace(inputs.profile, sql_analysis_dialect=dialect), ) -def completed_natively(inputs: SemanticInputs) -> CompiledProject: +def completed_natively(inputs: SemanticInputs) -> CompiledProjectView: """The native stage's completed project.""" return native_completion(inputs) -def note_kinds(project: CompiledProject, kinds: tuple[str, ...]) -> Counter[str]: +def note_kinds(project: CompiledProjectView, kinds: tuple[str, ...]) -> Counter[str]: """How many notes of each kind the project's diagnostics carry.""" notes: list[str] = list(chain.from_iterable(item.notes for item in project.diagnostics)) @@ -244,7 +246,7 @@ def note_kinds(project: CompiledProject, kinds: tuple[str, ...]) -> Counter[str] return counts -def native_completion(inputs: SemanticInputs) -> CompiledProject: +def native_completion(inputs: SemanticInputs) -> CompiledProjectView: """The native stage's completed project.""" return complete_native_semantic_diagnostics( @@ -268,7 +270,7 @@ def session_corpus_files(*, corpus: str, rng: random.Random, model_count: int) - return generators[corpus]() -def proven_output_count(project: CompiledProject) -> int: +def proven_output_count(project: CompiledProjectView) -> int: """How many models a dynamic pivot proof gave their output names.""" return sum( @@ -297,7 +299,7 @@ def recorded(**arguments: Any) -> frozenset[str]: return selections -def completion_view(project: CompiledProject) -> tuple[object, ...]: +def completion_view(project: CompiledProjectView) -> tuple[object, ...]: """Everything the stage decides: diagnostics and each model's binding and type facts.""" return ( @@ -306,7 +308,7 @@ def completion_view(project: CompiledProject) -> tuple[object, ...]: ) -def _model_view(model: CompiledModel) -> tuple[object, ...]: +def _model_view(model: CompiledModelView) -> tuple[object, ...]: return ( model.name, model.binding_diagnostics, @@ -558,9 +560,10 @@ def native_metadata(inputs: SemanticInputs) -> tuple[CompilerDiagnostic, ...]: def _family_counts(outcome: Any) -> Counter[str]: counts: Counter[str] = Counter() - for function_rows, reference_rows in outcome[0]: + for function_rows, reference_rows, schema_audit_rows in outcome[0]: counts.update(f"function {row[0]}" for row in function_rows) counts.update(f"reference {row[0]}" for row in reference_rows) + counts.update(f"schema_audit {row[0]}" for row in schema_audit_rows) counts.update(f"source {row[0]}" for _, row in outcome[1]) counts.update(f"sql_test {row[0]}" for _, row, _ in outcome[2]) return counts diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 41e10deb77..7fff6a5b7e 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -10,7 +10,8 @@ import pytest import sqlbuild._native as native_module -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, @@ -91,7 +92,7 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ ) views: dict[str, object] = {} for dialect in test_case.dialects: - native: CompiledProject = completed_natively(with_dialect(captured, dialect)) + native: CompiledProjectView = completed_natively(with_dialect(captured, dialect)) codes.update(item.code for item in native.diagnostics) codes.update(note_kinds(native, _NOTE_KINDS)) views[dialect] = completion_view(native) @@ -210,7 +211,7 @@ def test_given_project_without_catalog_when_completing_then_raises_native_compil with pytest.raises(native_module.NativeCompilerError) as raised: _ = native_completion( - replace(captured, project=replace(captured.project, binding_catalog=None)) + replace(captured, project=project_with(project=captured.project, binding_catalog=None)) ) assert str(raised.value) == test_case.expected_message @@ -267,7 +268,7 @@ def test_given_preview_compiles_when_completing_from_the_session_then_matches_pa payload_views: list[tuple[object, ...]] = [] session_views: list[tuple[object, ...]] = [] for index, inputs in enumerate(captured): - completed: CompiledProject = completed_natively(inputs) + completed: CompiledProjectView = completed_natively(inputs) codes.update(item.code for item in completed.diagnostics) codes.update(note_kinds(completed, _NOTE_KINDS)) session_views.append(completion_view(completed)) diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py index f7f45a42ef..28207b6a74 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py @@ -24,13 +24,3 @@ class EdgeSqlTestAssemblyTestCase: description: str files: dict[str, str] expected_outcome: str - - -@dataclass(frozen=True) -class HelperReadsTestCase: - """CTE bodies whose helper-key reads Python's `_parsed_reads` recorded in a golden.""" - - description: str - keys: frozenset[str] - bodies: tuple[str, ...] - expected_differences: list[tuple[int, str, object, object]] diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index 21f55641e9..859b4d7464 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -16,10 +16,9 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile._helpers.native_stages import sql_tests as sql_test_stage from sqlbuild.compiler.compile.models import ( - CompiledProject, CompiledSqlTest, - CompileProjectInputs, ) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.sql_test_glue.models import ( @@ -458,7 +457,7 @@ def assembly_outcome(*, project_dir: Path) -> PlanningCallOutcome: def _compiled_tests_and_diagnostics(*, project_dir: Path) -> tuple[object, object]: - project: CompiledProject = build_project_graph( + project: CompiledProjectView = build_project_graph( discovered_inputs=discover_project_inputs(project_dir=project_dir), adapter=DuckDbAdapter(), ).project @@ -485,7 +484,7 @@ def counted_failures(request: NativeSqlTestAssemblyRequest) -> list[NativeSqlTes answers.update(f"failed_{failure[0]}" for failure in filter(None, failures)) return rows - def counted_assemblies(*, inputs: CompileProjectInputs) -> tuple[NativeSqlTestAssembly, ...]: + def counted_assemblies(*, inputs: CompileInputsView) -> tuple[NativeSqlTestAssembly, ...]: assemblies: tuple[NativeSqlTestAssembly, ...] = seam(inputs=inputs) tests: list[CompiledSqlTest | None] = [item.test for item in assemblies] native: list[CompiledSqlTest] = list(filter(None, tests)) @@ -526,10 +525,3 @@ def _outcome(call: Callable[[], object]) -> PlanningCallOutcome: return PlanningCallOutcome(value=call(), raised=None) except Exception as error: # noqa: BLE001 - both paths must raise the same error return PlanningCallOutcome(value=None, raised=(type(error).__name__, str(error))) - - -def helper_reads_view(sql: str, keys: frozenset[str]) -> tuple[str, ...] | None: - """The native helper reads of `sql` as Python's `_parsed_reads` returned them.""" - - reads: list[str] | None = native_module.sql_test_helper_reads(sql, sorted(keys)) - return {True: lambda: None, False: lambda: tuple(cast(list[str], reads))}[reads is None]() diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_helper_reads.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_helper_reads.py deleted file mode 100644 index 7e12a8a639..0000000000 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_helper_reads.py +++ /dev/null @@ -1,60 +0,0 @@ -"""SQL-test helper CTE reads come from the native parse and match Python's `_parsed_reads`.""" - -from __future__ import annotations - -import pytest - -from tests.integration.src.sqlbuild.compiler.helpers import ( - GoldenEntry, - golden_differences, - golden_entry, - golden_name, - read_golden, -) -from tests.integration.src.sqlbuild.compiler.sql_test_glue._test_types import HelperReadsTestCase -from tests.integration.src.sqlbuild.compiler.sql_test_glue.helpers import helper_reads_view - - -@pytest.mark.parametrize( - "test_case", - [ - HelperReadsTestCase( - description="helper cte read edges", - keys=frozenset({"base", "helper", "mock_orders", "étapé"}), - bodies=( - "SELECT * FROM base", - "SELECT * FROM BASE JOIN helper h ON 1=1 JOIN base b2 ON 1=1", - "SELECT * FROM s.base JOIN db.s.helper ON 1=1", - "WITH base AS (SELECT 1) SELECT * FROM base, helper", - "SELECT * FROM (WITH Helper AS (SELECT 1) SELECT * FROM helper) x JOIN base ON 1=1", - "SELECT 1", - "SELECT * FROM unrelated", - "SELECT * FROM base WHERE (", - "SELECT * FROM ÉTAPÉ", - 'SELECT * FROM "Base"', - "SELECT * FROM mock_orders UNION ALL SELECT * FROM helper WHERE id IN (SELECT id FROM base)", - "SELECT base FROM other", - "SELECT * FROM helper; SELECT * FROM base", - "SELECT * FROM t WHERE EXISTS (SELECT 1 FROM helper)", - ), - expected_differences=[], - ), - ], - ids=lambda case: case.description, -) -def test_given_cte_bodies_when_reading_helper_keys_then_matches_python_golden( - test_case: HelperReadsTestCase, -) -> None: - views: list[tuple[str, ...] | None] = [ - helper_reads_view(body, test_case.keys) for body in test_case.bodies - ] - expected: list[GoldenEntry] = read_golden(golden_name("helper_reads", test_case.description)) - - assert ( - golden_differences(expected, [golden_entry("helper_reads", view) for view in views]) - == test_case.expected_differences - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py index d686ba683f..f0a8e3f9af 100644 --- a/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py +++ b/tests/integration/src/sqlbuild/executor/diff/test_query_artifact_cleanup.py @@ -15,6 +15,7 @@ from sqlbuild.cli.commands._helpers.diff.execution import execute_query_diff from sqlbuild.cli.commands.models import DiffCommandRequest, QueryDiffPreparation from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_QUERY_DIFF_ARTIFACT from sqlbuild.executor.diff.classes.query_artifact_lifecycle import QueryDiffArtifactLifecycle from sqlbuild.executor.diff.models import QueryDiffArtifact @@ -84,7 +85,7 @@ def test_given_expired_owned_and_untracked_artifacts_when_janitor_runs_then_only normalized_owned_name: str = owned.name.lower() connection.execute(f"ALTER TABLE {owned.relation} RENAME TO {normalized_owned_name}") connection.execute(f"CREATE TABLE {untracked.relation} AS SELECT 2 AS order_id") - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="run-1", effective_target_name="dev", effective_connection={}, diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py index fbb04bb973..1d20d66dff 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_graph.py @@ -7,10 +7,7 @@ from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import ( - CompiledProject, - CompileProjectInputs, -) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt._helpers.graph.core import ( @@ -80,14 +77,14 @@ def test_given_real_dbt_manifest_and_sqlbuild_project_when_building_graph_then_e discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( project_dir=sqlbuild_project_dir ) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, external_sql_reference_resolver=build_external_sql_reference_resolver( manifest_source=manifest_source ), ) - project: CompiledProject = assemble_compiled_project(inputs=compile_inputs) + project: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) manifest: DbtManifestIndex = build_dbt_manifest_index( raw_data=json.loads(manifest_source.read_text(encoding="utf-8")) ) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py index ed7eef6862..2d80ea5cec 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/test_manifest.py @@ -5,7 +5,7 @@ import pytest from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner @@ -67,7 +67,7 @@ def test_given_real_dbt_manifest_when_compiling_sqlbuild_then_preserves_dbt_ref( discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( project_dir=sqlbuild_project_dir ) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, external_sql_reference_resolver=build_external_sql_reference_resolver( diff --git a/tests/integration/src/sqlbuild/lint/main/helpers.py b/tests/integration/src/sqlbuild/lint/main/helpers.py index de2fac8de3..0a4772bf8f 100644 --- a/tests/integration/src/sqlbuild/lint/main/helpers.py +++ b/tests/integration/src/sqlbuild/lint/main/helpers.py @@ -6,14 +6,18 @@ from pathlib import Path from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project _PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' -def compile_models(*, project_dir: Path, models: tuple[tuple[str, str], ...]) -> CompiledProject: +def compile_models( + *, project_dir: Path, models: tuple[tuple[str, str], ...] +) -> CompiledProjectView: """Write and compile the models.""" (project_dir / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") @@ -26,10 +30,13 @@ def compile_models(*, project_dir: Path, models: tuple[tuple[str, str], ...]) -> ) -def without_lineage(project: CompiledProject) -> CompiledProject: +def without_lineage(project: CompiledProjectView) -> CompiledProjectView: """The same compilation with column lineage unavailable for every model.""" - return replace( - project, - models=tuple(replace(model, fast_lineage_columns=None) for model in project.models), + return project_with( + project=project, + models=tuple( + replace(compiled_model_record(model), fast_lineage_columns=None) + for model in project.models + ), ) diff --git a/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py b/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py index f3cba35506..d4a6746ecc 100644 --- a/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py +++ b/tests/integration/src/sqlbuild/lint/main/test_fix_verdict.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.lint.main.compile_facts import compile_facts from sqlbuild.lint.main.fix_verdict import fix_verdict from sqlbuild.lint.models import FixVerdict @@ -80,8 +80,12 @@ def test_given_baseline_and_fixed_compiles_when_comparing_then_failures_belong_t after_dir: Path = tmp_path / "after" before_dir.mkdir() after_dir.mkdir() - before: CompiledProject = compile_models(project_dir=before_dir, models=test_case.before_models) - after: CompiledProject = compile_models(project_dir=after_dir, models=test_case.after_models) + before: CompiledProjectView = compile_models( + project_dir=before_dir, models=test_case.before_models + ) + after: CompiledProjectView = compile_models( + project_dir=after_dir, models=test_case.after_models + ) verdict: FixVerdict = fix_verdict( before=compile_facts(project=before, project_dir=before_dir), @@ -119,10 +123,10 @@ def test_given_lineage_unavailable_on_both_sides_when_comparing_then_matching_co after_dir: Path = tmp_path / "after" before_dir.mkdir() after_dir.mkdir() - before: CompiledProject = without_lineage( + before: CompiledProjectView = without_lineage( compile_models(project_dir=before_dir, models=test_case.before_models) ) - after: CompiledProject = without_lineage( + after: CompiledProjectView = without_lineage( compile_models(project_dir=after_dir, models=test_case.after_models) ) diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index aef813813c..f3901cd630 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -7,6 +7,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.rule_engine._helpers.engine import custom_rules, fact_replay @@ -55,7 +56,7 @@ def resolve_reference( return None -def compile_files(*, root: Path, files: dict[str, str]) -> CompiledProject: +def compile_files(*, root: Path, files: dict[str, str]) -> CompiledProjectView: """Write and compile one DuckDB project with an external reference resolver.""" for relative_path, contents in files.items(): @@ -70,13 +71,13 @@ def compile_files(*, root: Path, files: dict[str, str]) -> CompiledProject: def dropped_fields(*, before: object, after: object) -> frozenset[str]: - """Return dataclass fields whose values differ between two snapshots of one object.""" + """Return the fields of `after`'s dataclass whose values differ between the two snapshots.""" - names: tuple[str, ...] = tuple(field.name for field in dataclasses.fields(type(before))) + names: tuple[str, ...] = tuple(field.name for field in dataclasses.fields(type(after))) return frozenset(filter(lambda name: getattr(before, name) != getattr(after, name), names)) -def every_fact_key(project: CompiledProject) -> tuple[FactKey, ...]: +def every_fact_key(project: CompiledProjectView) -> tuple[FactKey, ...]: """Return a key for every fact Rules can read, over every model of the project.""" models: tuple[Model, ...] = tuple(public_model(model) for model in project.models) @@ -94,7 +95,7 @@ def every_fact_key(project: CompiledProject) -> tuple[FactKey, ...]: ) -def fact_digests(*, project: CompiledProject, root: Path) -> dict[FactKey, tuple[str, str]]: +def fact_digests(*, project: CompiledProjectView, root: Path) -> dict[FactKey, tuple[str, str]]: """Digest every fact as hosts observe it and as whole-project digests encode it.""" views: RuleFactViews = build_rule_fact_views( @@ -112,7 +113,7 @@ def fact_digests(*, project: CompiledProject, root: Path) -> dict[FactKey, tuple def record_trimmed_fields( monkeypatch: pytest.MonkeyPatch, ) -> dict[type[object], set[str]]: - """Record which fields the host-payload trimmers replace, by dataclass type.""" + """Record replaced fields and omitted constructor fields in the host-payload trimmers.""" trimmed: dict[type[object], set[str]] = {CompiledProject: set(), CompiledModel: set()} @@ -120,6 +121,13 @@ def recording_replace(obj: Any, /, **changes: object) -> Any: trimmed.setdefault(type(obj), set()).update(changes) return dataclasses.replace(obj, **changes) + def recording_model(**values: Any) -> CompiledModel: + trimmed[CompiledModel].update( + {field.name for field in dataclasses.fields(CompiledModel)} - values.keys() + ) + return CompiledModel(**values) + monkeypatch.setattr(custom_rules, "replace", recording_replace) monkeypatch.setattr(custom_host_pool, "replace", recording_replace) + monkeypatch.setattr(custom_host_pool, "CompiledModel", recording_model) return trimmed diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py index 8b65e73896..ebec9b591a 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py @@ -5,6 +5,10 @@ import pytest from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_project_record import ( + compiled_project_record, +) from sqlbuild.rule_engine._helpers.engine.custom_rules import host_project from sqlbuild.rule_engine._helpers.host.custom_host_pool import host_payload_project from sqlbuild.rule_engine.classes.project_tree import public_model @@ -108,9 +112,9 @@ def test_given_compiled_project_when_trimming_host_payload_then_every_fact_is_unchanged( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostPayloadFactsTestCase ) -> None: - project: CompiledProject = compile_files(root=tmp_path, files=test_case.project_files) + project: CompiledProjectView = compile_files(root=tmp_path, files=test_case.project_files) trimmed_fields: dict[type[object], set[str]] = record_trimmed_fields(monkeypatch) - trimmed: CompiledProject = host_payload_project(host_project(project)) + trimmed: CompiledProjectView = host_payload_project(host_project(project)) full_digests: dict[FactKey, tuple[str, str]] = fact_digests(project=project, root=tmp_path) trimmed_digests: dict[FactKey, tuple[str, str]] = fact_digests(project=trimmed, root=tmp_path) @@ -118,7 +122,7 @@ def test_given_compiled_project_when_trimming_host_payload_then_every_fact_is_un assert trimmed_digests == full_digests assert tuple(map(public_model, trimmed.models)) == tuple(map(public_model, project.models)) assert ( - dropped_fields(before=project, after=trimmed) + dropped_fields(before=compiled_project_record(project), after=trimmed) == trimmed_fields[CompiledProject] - test_case.expected_unpopulated_trimmed_fields ) assert ( diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py new file mode 100644 index 0000000000..392d12545c --- /dev/null +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py @@ -0,0 +1,31 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeRulesRequestTestCase: + """A compile of the rules fixture: request rows built natively and cache use.""" + + description: str + expected_built_rows: tuple[int, int, int] + expected_cold_cache: tuple[int, int] + expected_warm_cache: tuple[int, int] + expected_edited_cache: tuple[int, int] + + +@dataclass(frozen=True) +class NativeTypeProofTestCase: + """A type-proof findings for a contract with matching and mismatched casts.""" + + description: str + expected_findings: tuple[tuple[str, str, int | None, int | None, str], ...] + expected_built_rows: tuple[int, int, int] + + +@dataclass(frozen=True) +class StructPassthroughTestCase: + """A STRUCT passthrough whose type proof asks the adapter's `types_equal` callback.""" + + description: str + declared_type: str + expected_exit_code: int + expected_codes: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py new file mode 100644 index 0000000000..22f4df7caa --- /dev/null +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py @@ -0,0 +1,136 @@ +import json +import shutil +from collections import Counter +from pathlib import Path +from typing import Any, cast + +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.rule_engine._helpers.run import native_rows as native_rows_module + +RULES_FIXTURE: Path = Path("tests/e2e/fixtures/waffle_shop") +RULES_CONFIG: str = '\n[rules]\nselect = ["SQBR"]\n' +EDITED_MODEL: str = "models/staging/stg_orders.sql" + +type FindingKey = tuple[str, str, int | None, int | None, str] + + +def write_rules_fixture(*, project_dir: Path) -> None: + """Copy the neutral fixture and select every built-in rule.""" + + _ = shutil.copytree(RULES_FIXTURE, project_dir) + config: Path = project_dir / "sqlbuild_project.toml" + _ = config.write_text(config.read_text(encoding="utf-8") + RULES_CONFIG, encoding="utf-8") + + +def record_built_rows(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: + """Count the rules request rows the native builder reports it built.""" + + built: Counter[str] = Counter() + + def counted(*, stage: NativeStage, kind: str, units: int = 1) -> None: + built[f"{stage.value}:{kind}"] += units + + monkeypatch.setattr(native_rows_module, "report_native_answer", counted) + return built + + +def built_rows(built: Counter[str]) -> tuple[int, int, int]: + """Models, SQL tests and scenarios built natively, in that order.""" + + return ( + built["rules_request:models"], + built["rules_request:sql_tests"], + built["rules_request:sql_scenarios"], + ) + + +def compile_rules( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[tuple[FindingKey, ...], tuple[int, int]]: + """Compile with the rules cache on; return findings in order and rule cache hits and misses.""" + + _ = main(["--project-dir", str(project_dir), "compile", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + diagnostics: list[dict[str, Any]] = cast(list[dict[str, Any]], payload["diagnostics"]) + timings: dict[str, int] = cast(dict[str, int], payload["compile_timings"]) + return ( + tuple( + (item["code"], item["path"], item.get("line"), item.get("column"), item["message"]) + for item in diagnostics + ), + (timings["rule_cache_hits"], timings["rule_cache_misses"]), + ) + + +def struct_passthrough_files(*, declared_type: str) -> dict[str, str]: + """An enforced passthrough of a STRUCT column whose contract declares `declared_type`.""" + + return { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' + '[defaults]\ncontract = "enforced"\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.',\n" + " columns (order_id (type INTEGER), detail (type 'STRUCT(\"café\" INTEGER)')),\n" + ");\n\n" + "SELECT CAST(1 AS INTEGER) AS order_id,\n" + " CAST({'café': 1} AS STRUCT(\"café\" INTEGER)) AS detail\n" + ), + "models/orders.sql": ( + "MODEL (description 'Typed orders.',\n" + f" columns (order_id (type INTEGER), detail (type '{declared_type}')),\n" + ");\n\n" + 'SELECT order_id, detail\nFROM __ref("stg_orders")\n' + ), + } + + +def compile_codes( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[int, tuple[str, ...]]: + """Compile to JSON; return the exit code and every diagnostic code in order.""" + + exit_code: int = main(["--project-dir", str(project_dir), "compile", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + diagnostics: list[dict[str, Any]] = cast(list[dict[str, Any]], payload["diagnostics"]) + return exit_code, tuple(item["code"] for item in diagnostics) + + +def edit_model(*, project_dir: Path) -> None: + """Change one model's SQL so only its rules facts change.""" + + model: Path = project_dir / EDITED_MODEL + _ = model.write_text(model.read_text(encoding="utf-8") + "\n-- reviewed\n", encoding="utf-8") + + +TYPE_PROOF_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' + '[defaults]\ncontract = "enforced"\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.',\n" + " columns (order_id (type INTEGER), amount (type INTEGER)),\n" + ");\n\n" + "SELECT CAST(1 AS INTEGER) AS order_id, CAST(5 AS INTEGER) AS amount\n" + ), + "models/orders.sql": ( + "MODEL (description 'Typed orders.',\n" + " columns (order_id (type INTEGER), amount (type BIGINT)),\n" + ");\n\n" + 'SELECT order_id, amount\nFROM __ref("stg_orders")\n' + ), +} + + +def write_files(*, project_dir: Path, files: dict[str, str]) -> None: + """Write project files at their project-relative paths.""" + + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_native_rules_request.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_native_rules_request.py new file mode 100644 index 0000000000..6f57da2198 --- /dev/null +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_native_rules_request.py @@ -0,0 +1,156 @@ +"""Integration coverage for the built-in rules request built natively from compiled rows.""" + +from collections import Counter +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.rule_engine.native_rows._test_types import ( + NativeRulesRequestTestCase, + NativeTypeProofTestCase, + StructPassthroughTestCase, +) +from tests.integration.src.sqlbuild.rule_engine.native_rows.helpers import ( + TYPE_PROOF_FILES, + FindingKey, + built_rows, + compile_codes, + compile_rules, + edit_model, + record_built_rows, + struct_passthrough_files, + write_files, + write_rules_fixture, +) + +_TYPE_PROOF_FINDINGS: tuple[FindingKey, ...] = ( + ( + "K002", + "models/orders.sql", + 2, + 37, + "column 'amount' inferred as INTEGER but declared type is BIGINT", + ), + ( + "SQBRCONTRACT105", + "models/orders.sql", + 1, + 18, + 'output "amount" is a passthrough whose type is not proven against the declared ' + "contract; declared type is BIGINT", + ), +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeRulesRequestTestCase( + description="compile builds every request row natively", + expected_built_rows=(39, 15, 9), + expected_cold_cache=(0, 26), + expected_warm_cache=(26, 0), + expected_edited_cache=(24, 2), + ), + ], + ids=lambda case: case.description, +) +def test_given_rules_fixture_when_compiling_cold_warm_and_edited_then_findings_match_fresh( + test_case: NativeRulesRequestTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + fresh_dir: Path = tmp_path / "fresh" + fresh_edited_dir: Path = tmp_path / "fresh_edited" + project_dir: Path = tmp_path / "project" + write_rules_fixture(project_dir=fresh_dir) + write_rules_fixture(project_dir=fresh_edited_dir) + edit_model(project_dir=fresh_edited_dir) + write_rules_fixture(project_dir=project_dir) + _ = record_built_rows(monkeypatch=monkeypatch) + fresh: tuple[FindingKey, ...] = compile_rules(project_dir=fresh_dir, capsys=capsys)[0] + fresh_edited: tuple[FindingKey, ...] = compile_rules( + project_dir=fresh_edited_dir, capsys=capsys + )[0] + built: Counter[str] = record_built_rows(monkeypatch=monkeypatch) + + cold: tuple[tuple[FindingKey, ...], tuple[int, int]] = compile_rules( + project_dir=project_dir, capsys=capsys + ) + warm: tuple[tuple[FindingKey, ...], tuple[int, int]] = compile_rules( + project_dir=project_dir, capsys=capsys + ) + edit_model(project_dir=project_dir) + edited: tuple[tuple[FindingKey, ...], tuple[int, int]] = compile_rules( + project_dir=project_dir, capsys=capsys + ) + + assert (cold[0], warm[0], edited[0]) == (fresh, fresh, fresh_edited) + assert (cold[1], warm[1], edited[1]) == ( + test_case.expected_cold_cache, + test_case.expected_warm_cache, + test_case.expected_edited_cache, + ) + assert built_rows(built) == test_case.expected_built_rows + + +@pytest.mark.parametrize( + "test_case", + [ + NativeTypeProofTestCase( + description="compile proves passthrough types natively", + expected_findings=_TYPE_PROOF_FINDINGS, + expected_built_rows=(2, 0, 0), + ), + ], + ids=lambda case: case.description, +) +def test_given_matching_and_mismatched_passthroughs_when_compiling_then_only_mismatch_is_found( + test_case: NativeTypeProofTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) + built: Counter[str] = record_built_rows(monkeypatch=monkeypatch) + + findings: tuple[FindingKey, ...] = compile_rules(project_dir=tmp_path, capsys=capsys)[0] + + assert findings == test_case.expected_findings + assert built_rows(built) == test_case.expected_built_rows + + +@pytest.mark.parametrize( + "test_case", + [ + StructPassthroughTestCase( + description="declared STRUCT equals the passthrough", + declared_type='STRUCT("café" INTEGER)', + expected_exit_code=0, + expected_codes=(), + ), + StructPassthroughTestCase( + description="declared STRUCT differs from the passthrough", + declared_type='STRUCT("café" BIGINT)', + expected_exit_code=1, + expected_codes=("K002", "SQBRCONTRACT105"), + ), + ], + ids=lambda case: case.description, +) +def test_given_struct_passthrough_when_proving_types_then_adapter_callback_decides( + test_case: StructPassthroughTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_files( + project_dir=tmp_path, + files=struct_passthrough_files(declared_type=test_case.declared_type), + ) + _ = record_built_rows(monkeypatch=monkeypatch) + + outcome: tuple[int, tuple[str, ...]] = compile_codes(project_dir=tmp_path, capsys=capsys) + + assert outcome == (test_case.expected_exit_code, test_case.expected_codes) diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py index 3bba3bdc8a..10eb9bf0d4 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py @@ -7,7 +7,7 @@ import pytest from scripts.compiler_differential._helpers.corpus.corpus import build_corpus -from scripts.compiler_differential.constants import COLD_COMPILE +from scripts.compiler_differential.constants import COLD_COMPILE, DIALECT_RICH_LINEAGE from scripts.compiler_differential.models import CorpusProject, ExpectedOutcome from tests.unit.scripts.compiler_differential._helpers.corpus._test_types import ( CorpusExpectationTestCase, @@ -79,12 +79,12 @@ def test_given_corpus_selection_when_building_then_entries_declare_expected_outc "seed/5-postgres", "seed/5-snowflake", ), - expected_variant_commands=(COLD_COMPILE,), + expected_variant_commands=(COLD_COMPILE, DIALECT_RICH_LINEAGE), ) ], ids=lambda case: case.description, ) -def test_given_seed_range_when_building_then_first_seed_adds_compile_only_dialect_variants( +def test_given_seed_range_when_building_then_first_seed_adds_compile_and_rich_lineage_variants( test_case: DialectVariantTestCase, ) -> None: corpus: list[CorpusProject] = build_corpus( diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py index d873fc67d9..70b36fb2ee 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py @@ -4,7 +4,7 @@ from dataclasses import dataclass -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -69,7 +69,7 @@ class RenderEncodedInputsTestCase: """Real compile inputs, encoded by the stage capture, and the render kinds they prove.""" description: str - inputs: CompileProjectInputs + inputs: CompileInputsView expected_kinds: frozenset[str] diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 9a71c56c7a..0d2a8caf65 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -42,11 +42,15 @@ def empty_inputs_capture(**overrides: object) -> dict[str, object]: def empty_render_capture(**overrides: object) -> dict[str, object]: - """Return a capture with every `CompileProjectInputs` field, in order, then overrides.""" + """Return every captured `CompileProjectInputs` field, in order, then overrides.""" + type_name: str = "sqlbuild.compiler.compile.models:CompileProjectInputs" + names: list[str] = [field.name for field in dataclasses.fields(CompileProjectInputs)] return { - "__type__": "sqlbuild.compiler.compile.models:CompileProjectInputs", - **{field.name: [] for field in dataclasses.fields(CompileProjectInputs)}, + "__type__": type_name, + **dict.fromkeys( + filterfalse(STAGE_CAPTURE_OMITTED_ATTRIBUTES[type_name].__contains__, names), [] + ), **overrides, } diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/test_native_fallbacks.py b/tests/unit/scripts/compiler_differential/_helpers/running/test_native_fallbacks.py index 60bf52a9f9..aa71c03f6a 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/running/test_native_fallbacks.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/test_native_fallbacks.py @@ -97,7 +97,40 @@ class _SwitchableStage(StrEnum): }, _SESSION: {"seed": 1}, }, - ) + ), + FallbackRecordsTestCase( + description="racy_timestamp_hash_counts_are_metrics_not_exact_ownership_counts", + project="seed/3", + records=( + AnalysisRecords( + fallbacks={ + ("compile_outputs.native", "reuse_digested_files"): 1, + ("compile_outputs.native", "reuse_replays"): 1, + } + ), + AnalysisRecords( + fallbacks={ + ("compile_outputs.native", "reuse_digested_files"): 1000, + ("compile_outputs.native", "reuse_replays"): 1, + ("compile_outputs.rescue", "reuse_digested_files"): 1, + } + ), + ), + expected_observed={ + ("native", "compile_outputs", "compile_outputs.native", "reuse_replays"): { + "seed": 1 + }, + ("native-preview", "compile_outputs", "compile_outputs.native", "reuse_replays"): { + "seed": 1 + }, + ( + "native-preview", + "compile_outputs", + "compile_outputs.rescue", + "reuse_digested_files", + ): {"seed": 1}, + }, + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/scripts/native_layering/_helpers/_test_types.py b/tests/unit/scripts/native_layering/_helpers/_test_types.py index e4884e89b7..468e6a1fbc 100644 --- a/tests/unit/scripts/native_layering/_helpers/_test_types.py +++ b/tests/unit/scripts/native_layering/_helpers/_test_types.py @@ -23,4 +23,5 @@ class NativeVersionTestCase: workspace_version: str pyproject_version: str release_version: str + cli_version: str expected_errors: tuple[str, ...] diff --git a/tests/unit/scripts/native_layering/_helpers/helpers.py b/tests/unit/scripts/native_layering/_helpers/helpers.py index 5b37019fa1..1508abe90b 100644 --- a/tests/unit/scripts/native_layering/_helpers/helpers.py +++ b/tests/unit/scripts/native_layering/_helpers/helpers.py @@ -43,12 +43,14 @@ def _lock_package(name: str, dependencies: tuple[str, ...]) -> str: def write_versions(root: Path, test_case: NativeVersionTestCase) -> None: - """Write the three release version sources the version check compares.""" + """Write the four release version sources the version check compares.""" (root / "Cargo.toml").write_text( f'[workspace.package]\nversion = "{test_case.workspace_version}"\n', encoding="utf-8" ) (root / "pyproject.toml").write_text( - f'[project]\nversion = "{test_case.pyproject_version}"\n', encoding="utf-8" + f'[project]\nversion = "{test_case.pyproject_version}"\n' + f'dependencies = ["sqlbuild-cli=={test_case.cli_version}"]\n', + encoding="utf-8", ) (root / ".release-please-manifest.json").write_text( f'{{".": "{test_case.release_version}"}}\n', encoding="utf-8" diff --git a/tests/unit/scripts/native_layering/_helpers/test_versions.py b/tests/unit/scripts/native_layering/_helpers/test_versions.py index 24652c0d13..563a8b3ad7 100644 --- a/tests/unit/scripts/native_layering/_helpers/test_versions.py +++ b/tests/unit/scripts/native_layering/_helpers/test_versions.py @@ -18,6 +18,7 @@ workspace_version="1.2.0", pyproject_version="1.2.0", release_version="1.2.0", + cli_version="1.2.0", expected_errors=(), ), ], @@ -39,10 +40,25 @@ def test_given_matching_versions_when_checking_then_returns_no_errors( workspace_version="1.1.9", pyproject_version="1.2.0", release_version="1.2.0", + cli_version="1.2.0", expected_errors=( "Release versions differ: Cargo.toml [workspace.package] version is 1.1.9, " "pyproject.toml [project] version is 1.2.0, " - ".release-please-manifest.json version is 1.2.0.", + ".release-please-manifest.json version is 1.2.0, " + "pyproject.toml sqlbuild-cli dependency is 1.2.0.", + ), + ), + NativeVersionTestCase( + description="native executable pin lags a release", + workspace_version="1.2.0", + pyproject_version="1.2.0", + release_version="1.2.0", + cli_version="1.1.9", + expected_errors=( + "Release versions differ: Cargo.toml [workspace.package] version is 1.2.0, " + "pyproject.toml [project] version is 1.2.0, " + ".release-please-manifest.json version is 1.2.0, " + "pyproject.toml sqlbuild-cli dependency is 1.1.9.", ), ), ], diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_run_context.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_run_context.py index 5365952a8d..13b9255c98 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_run_context.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_run_context.py @@ -18,7 +18,12 @@ from sqlbuild.cli.entry.models import ( SelectorFileSummary, ) -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompiledSource +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledSourceView, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredTaskFunction from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( @@ -110,16 +115,16 @@ def test_given_resolved_build_context_when_writing_then_renders_safe_verbose_summary( test_case: BuildRunContextTestCase, ) -> None: - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="20260830T175430Z_fb5fb40a8b81", effective_target_name="test", effective_connection={}, effective_vars=test_case.effective_vars, effective_target_database="ANALYTICS", effective_target_schema="DEV_DAGSTER_DEV", - models=cast(tuple[CompiledModel, ...], (object(), object())), + models=cast(tuple[CompiledModelView, ...], (object(), object())), sources=cast( - tuple[CompiledSource, ...], + tuple[CompiledSourceView, ...], tuple( SimpleNamespace(source_entry=SimpleNamespace(loader="load_orders")) for _ in range(test_case.total_source_count) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/_test_types.py index 0a8e703a11..60360ccf56 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/_test_types.py @@ -134,3 +134,36 @@ class SharedSelectorDepthTestCase: select: tuple[str, ...] direction: str | None expected_node_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class FingerprintedFilesTestCase: + """Authored files whose relation lineage fingerprint hashes exactly the expected inputs.""" + + description: str + files: dict[str, str] + links: dict[str, str] + unreadable_directories: tuple[str, ...] + environment: dict[str, str] + cli_vars: dict[str, object] | None + expected_hashed_files: tuple[str, ...] + expected_environment_names: tuple[str, ...] + + +@dataclass(frozen=True) +class UncacheableFingerprintTestCase: + """Authored files whose relation lineage fingerprint is unavailable.""" + + description: str + files: dict[str, str] + expected_fingerprint: None + + +@dataclass(frozen=True) +class InterruptedListingFingerprintTestCase: + """A directory whose listing fails after it opened, under this Python's own `rglob`.""" + + description: str + files: dict[str, str] + interrupted_directory: str + expected_uncacheable_before: tuple[int, int] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/helpers.py index 4a1066d858..039fc21d05 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/helpers.py @@ -2,8 +2,16 @@ from __future__ import annotations -from collections.abc import Iterable +import glob +import hashlib +import inspect +import os +from collections.abc import Callable, Iterable, Iterator from pathlib import Path +from types import TracebackType +from typing import Protocol + +import pytest from sqlbuild.cli.commands.models import ColumnLineageTrace, LineageNode from sqlbuild.compiler.compile.models import ( @@ -16,7 +24,11 @@ CompileModelConfig, InferredColumn, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.discovery.models import ( DiscoveredSchemaFile, DiscoveredSeedFile, @@ -26,6 +38,7 @@ from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, ColumnTransformKind from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.spec.contracts.models import SchemaSeedEntry, SettingsConfig, SourceEntry +from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes def build_lineage_test_graph() -> ProjectGraph: @@ -50,7 +63,7 @@ def build_lineage_test_graph() -> ProjectGraph: model_entries=(), seed_entries=(), ) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="test-run", effective_target_name=None, effective_connection={}, @@ -123,7 +136,7 @@ def build_lineage_test_graph() -> ProjectGraph: fact_orders_key: (daily_rollup_key,), daily_rollup_key: (), } - return ProjectGraph( + return project_graph_from_indexes( project=project, upstream_deps=upstream_deps, downstream_deps=downstream_deps, @@ -225,7 +238,7 @@ def _model( relative_path: str, query_sql: str = "SELECT 1", inferred_columns: tuple[str, ...] = (), -) -> CompiledModel: +) -> CompiledModelView: return CompiledModel( key=key, deps=deps, @@ -245,3 +258,84 @@ def _target(name: str) -> CompiledRelationLocation: name=name, qualified_name=f"main.{name}", ) + + +class _ScandirListing(Protocol): + """The part of `os.scandir`'s iterator the interrupted listing wraps.""" + + def close(self) -> None: ... + + +class _InterruptedListing: + """A `scandir` iterator whose first step fails after the directory opened.""" + + def __init__(self, listing: _ScandirListing) -> None: + self._listing: _ScandirListing = listing + + def __enter__(self) -> _InterruptedListing: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self._listing.close() + + def __iter__(self) -> Iterator[os.DirEntry[str]]: + return self + + def __next__(self) -> os.DirEntry[str]: + raise OSError(5, "Input/output error") + + def close(self) -> None: + self._listing.close() + + +class _NoStringGlobber: + """Stands in for `glob._StringGlobber` on Pythons whose `rglob` does not use it.""" + + +def interrupt_listing(*, monkeypatch: pytest.MonkeyPatch, directory: Path) -> None: + """Make every `scandir` of `directory` fail mid-listing, wherever glob binds `scandir`.""" + + real_scandir: Callable[..., _ScandirListing] = os.scandir + target: str = os.path.abspath(directory) + + def scandir(path: str | os.PathLike[str] = ".") -> object: + listing: _ScandirListing = real_scandir(path) + choices: dict[bool, object] = {True: _InterruptedListing(listing), False: listing} + return choices[os.path.abspath(os.fspath(path)) == target] + + globber: object = getattr(glob, "_StringGlobber", _NoStringGlobber) + bound_early: bool = getattr(globber, "scandir", None) is real_scandir + replacements: dict[bool, object] = { + True: staticmethod(scandir), + False: inspect.getattr_static(globber, "scandir", None), + } + monkeypatch.setattr(os, "scandir", scandir) + monkeypatch.setattr(globber, "scandir", replacements[bound_early], raising=False) + + +def expected_fingerprint( + *, + project_dir: Path, + prefix: bytes, + hashed_files: tuple[str, ...], + environment_names: tuple[str, ...], +) -> str: + """The cache key over exactly `hashed_files`, in this platform's path order.""" + + digest: hashlib._Hash = hashlib.sha256(prefix) + for path in sorted(project_dir / relative_path for relative_path in hashed_files): + relative_path: str = path.relative_to(project_dir).as_posix() + contents: bytes = path.read_bytes() + digest.update(len(relative_path).to_bytes(8, byteorder="big")) + digest.update(relative_path.encode("utf-8")) + digest.update(len(contents).to_bytes(8, byteorder="big")) + digest.update(contents) + for name in sorted(environment_names): + digest.update(name.encode("ascii")) + digest.update(os.environ.get(name, "").encode("utf-8")) + return digest.hexdigest() diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py index 5d8df636e4..1e54dbf9c9 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/lineage/test_cache.py @@ -1,14 +1,71 @@ from __future__ import annotations +import contextlib +import sys from pathlib import Path import pytest -from sqlbuild.cli.commands._helpers.lineage.cache import relation_lineage_fingerprint +from sqlbuild.cli.commands._helpers.lineage.cache import ( + _INTERRUPTED_LISTING_UNCACHEABLE, + _fingerprint_prefix, + relation_lineage_fingerprint, +) from tests.unit.src.sqlbuild.cli.commands._helpers.lineage._test_types import ( + FingerprintedFilesTestCase, + InterruptedListingFingerprintTestCase, LineageFingerprintAvailabilityTestCase, LineageFingerprintEnvironmentTestCase, + UncacheableFingerprintTestCase, +) +from tests.unit.src.sqlbuild.cli.commands._helpers.lineage.helpers import ( + expected_fingerprint, + interrupt_listing, +) + +_LAYOUT_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\nschema = "${ENV:ORDERS_SCHEMA}"\n', + "models/a/b.sql": "SELECT 1 AS order_id\n", + "models/a-c.sql": "SELECT '${ENV: ORDERS_REGION}' AS region\n", + "models/a.sql": "SELECT 2 AS order_id\n", + "models/Z.SQL": "SELECT 3 AS order_id\n", + "models/é.sql": "SELECT 'café' AS label\n", + "seeds/customers.csv": "customer_id\n1\n", + "sources/raw.YML": "sources: []\n", + "macros/helpers.py": "def total():\n return 1\n", + ".hidden/notes.yaml": "a: 1\n", + ".gitignore": "target/\n", + "nested/.sqlbuildignore": "*.tmp\n", + ".sql": "SELECT 'not a suffix'\n", + "README.md": "${CTX:ignored}\n", + "noext": "ENV:\n", + "target/ignored.sql": "ENV:\n", + ".venv/ignored.py": "ENV:\n", + "models/__pycache__/ignored.py": "ENV:\n", + "nested/target/kept.sql": "SELECT 4 AS order_id\n", +} +_LAYOUT_LINKS: dict[str, str] = { + "models/linked.sql": "models/a.sql", + "linked_dir": "models", + "models/broken.sql": "models/missing.sql", +} +_LAYOUT_UNLINKED_HASHED_FILES: tuple[str, ...] = ( + ".gitignore", + ".hidden/notes.yaml", + "macros/helpers.py", + "models/Z.SQL", + "models/a/b.sql", + "models/a-c.sql", + "models/a.sql", + "models/é.sql", + "nested/.sqlbuildignore", + "nested/target/kept.sql", + "seeds/customers.csv", + "sources/raw.YML", + "sqlbuild_project.toml", ) +_LAYOUT_HASHED_FILES: tuple[str, ...] = (*_LAYOUT_UNLINKED_HASHED_FILES, "models/linked.sql") +_LAYOUT_ENVIRONMENT_NAMES: tuple[str, ...] = ("ORDERS_REGION", "ORDERS_SCHEMA") @pytest.mark.parametrize( @@ -87,3 +144,165 @@ def test_given_dynamic_invocation_context_when_fingerprinting_then_disables_cach observed: str | None = relation_lineage_fingerprint(project_dir=tmp_path, cli_vars=None) assert (observed is not None) is test_case.expected_available + + +@pytest.mark.parametrize( + "test_case", + ( + FingerprintedFilesTestCase( + description="sorting, suffixes, exclusions, links and environment values", + files=_LAYOUT_FILES, + links=_LAYOUT_LINKS, + unreadable_directories=(), + environment={"ORDERS_SCHEMA": "orders_dev"}, + cli_vars={"region": "north", "limits": [1, 2.5, None], "label": "café"}, + expected_hashed_files=_LAYOUT_HASHED_FILES, + expected_environment_names=_LAYOUT_ENVIRONMENT_NAMES, + ), + FingerprintedFilesTestCase( + description="chmod permissions follow the platform directory semantics", + files={**_LAYOUT_FILES, "models/locked/hidden.sql": "SELECT 5 AS order_id\n"}, + links={}, + unreadable_directories=("models/locked",), + environment={"ORDERS_SCHEMA": "orders_dev"}, + cli_vars=None, + expected_hashed_files=_LAYOUT_UNLINKED_HASHED_FILES + + (("models/locked/hidden.sql",) if sys.platform == "win32" else ()), + expected_environment_names=_LAYOUT_ENVIRONMENT_NAMES, + ), + FingerprintedFilesTestCase( + description="undecodable names that are never hashed", + files={ + **_LAYOUT_FILES, + "models/notes_\udcff.md": "ENV:\n", + "models/\udcfe_dir/readme.txt": "ENV:\n", + }, + links={}, + unreadable_directories=(), + environment={"ORDERS_SCHEMA": "orders_dev"}, + cli_vars=None, + expected_hashed_files=_LAYOUT_UNLINKED_HASHED_FILES, + expected_environment_names=_LAYOUT_ENVIRONMENT_NAMES, + ), + FingerprintedFilesTestCase( + description="an undecodable hashed file inside an excluded directory", + files={**_LAYOUT_FILES, "target/orders_\udcff.sql": "SELECT 8 AS order_id\n"}, + links={}, + unreadable_directories=(), + environment={"ORDERS_SCHEMA": "orders_dev"}, + cli_vars=None, + expected_hashed_files=_LAYOUT_UNLINKED_HASHED_FILES, + expected_environment_names=_LAYOUT_ENVIRONMENT_NAMES, + ), + ), + ids=lambda case: case.description, +) +def test_given_authored_files_when_fingerprinting_then_hashes_exactly_the_expected_inputs( + test_case: FingerprintedFilesTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + for relative_path, contents in test_case.files.items(): + path: Path = tmp_path / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + for relative_path, target in test_case.links.items(): + (tmp_path / relative_path).symlink_to(tmp_path / target) + for relative_path in test_case.unreadable_directories: + (tmp_path / relative_path).chmod(0) + for name, value in test_case.environment.items(): + monkeypatch.setenv(name, value) + monkeypatch.delenv("ORDERS_REGION", raising=False) + + observed: str | None = relation_lineage_fingerprint( + project_dir=tmp_path, cli_vars=test_case.cli_vars + ) + + for relative_path in test_case.unreadable_directories: + (tmp_path / relative_path).chmod(0o755) + assert observed == expected_fingerprint( + project_dir=tmp_path, + prefix=_fingerprint_prefix(cli_vars=test_case.cli_vars), + hashed_files=test_case.expected_hashed_files, + environment_names=test_case.expected_environment_names, + ) + + +@pytest.mark.parametrize( + "test_case", + ( + UncacheableFingerprintTestCase( + description="an environment marker without a name", + files={**_LAYOUT_FILES, "models/bad.sql": "SELECT 'ENV: ' AS broken\n"}, + expected_fingerprint=None, + ), + UncacheableFingerprintTestCase( + description="an environment name followed by a non-ASCII byte", + files={"models/bad.sql": "SELECT '${ENV:ORDERSÉ}' AS broken\n"}, + expected_fingerprint=None, + ), + UncacheableFingerprintTestCase( + description="overlapping markers inside one matched name", + files={"models/bad.sql": "SELECT 'ENV: ENV:X' AS broken\n"}, + expected_fingerprint=None, + ), + UncacheableFingerprintTestCase( + description="dynamic context in Python source", + files={"macros/run.py": "RUN = 'CTX:run_id'\n", "models/a.sql": "SELECT 1\n"}, + expected_fingerprint=None, + ), + UncacheableFingerprintTestCase( + description="an undecodable hashed file name", + files={**_LAYOUT_FILES, "models/orders_\udcff.sql": "SELECT 6 AS order_id\n"}, + expected_fingerprint=None, + ), + UncacheableFingerprintTestCase( + description="a hashed file below an undecodable directory", + files={**_LAYOUT_FILES, "models/\udcfe_dir/orders.sql": "SELECT 7 AS order_id\n"}, + expected_fingerprint=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_unhashable_inputs_when_fingerprinting_then_the_cache_is_disabled( + test_case: UncacheableFingerprintTestCase, tmp_path: Path +) -> None: + for relative_path, contents in test_case.files.items(): + path: Path = tmp_path / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + observed: str | None = relation_lineage_fingerprint(project_dir=tmp_path, cli_vars=None) + + assert observed is test_case.expected_fingerprint + + +@pytest.mark.parametrize( + "test_case", + ( + InterruptedListingFingerprintTestCase( + description="a hashed directory whose listing fails mid-way", + files={"sqlbuild_project.toml": 'name = "orders"\n', "models/a.sql": "SELECT 1\n"}, + interrupted_directory="models", + expected_uncacheable_before=(3, 13), + ), + ), + ids=lambda case: case.description, +) +def test_given_interrupted_listing_when_python_globs_then_native_policy_matches_rglob( + test_case: InterruptedListingFingerprintTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + for relative_path, contents in test_case.files.items(): + path: Path = tmp_path / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + interrupt_listing(monkeypatch=monkeypatch, directory=tmp_path / test_case.interrupted_directory) + listed: list[Path] = [] + + with contextlib.suppress(OSError): + listed = list(tmp_path.rglob("*")) + + assert (not listed) is (sys.version_info < test_case.expected_uncacheable_before) + assert (not listed) is _INTERRUPTED_LISTING_UNCACHEABLE diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py index 8f9d5615da..05a0405892 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/_test_types.py @@ -58,3 +58,10 @@ class CommandLineQuotingTestCase: class LayerFolderMappingTestCase: description: str expected_layers: frozenset[str] + + +@dataclass(frozen=True) +class ImportBoundaryTestCase: + description: str + imported_name: str + expected_modules: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py index 9f9a39a231..8b20e1afbe 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/helpers.py @@ -2,6 +2,8 @@ from __future__ import annotations +import ast +from itertools import compress from pathlib import Path, PurePosixPath from sqlbuild.compiler.compile.models import CompilerDiagnostic @@ -60,3 +62,19 @@ def rule_diagnostics(*, codes: tuple[str, ...], path: str) -> tuple[CompilerDiag ) for code in codes ) + + +def modules_importing(*, package_dir: Path, name: str) -> list[str]: + """Return the sorted file names of the package's modules that import `name`.""" + + paths: list[Path] = sorted(package_dir.glob("*.py")) + flags: list[bool] = [_imports(path=path, name=name) for path in paths] + return [path.name for path in compress(paths, flags)] + + +def _imports(*, path: Path, name: str) -> bool: + tree: ast.Module = ast.parse(path.read_text(encoding="utf-8")) + imported: set[tuple[type[ast.AST], object]] = { + (type(node), getattr(node, "name", None)) for node in ast.walk(tree) + } + return (ast.alias, name) in imported diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_facts_boundary.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_facts_boundary.py new file mode 100644 index 0000000000..b5ee4a8b80 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/refactor/test_facts_boundary.py @@ -0,0 +1,38 @@ +"""Only the facts module of `sqb rename` and `sqb mv` reads `CompiledProject`.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import sqlbuild.cli.commands._helpers.refactor as refactor_package +from tests.unit.src.sqlbuild.cli.commands._helpers.refactor._test_types import ( + ImportBoundaryTestCase, +) +from tests.unit.src.sqlbuild.cli.commands._helpers.refactor.helpers import modules_importing + + +@pytest.mark.parametrize( + "test_case", + [ + ImportBoundaryTestCase( + description="the native command core will return these facts instead", + imported_name="CompiledProjectView", + expected_modules=("facts.py",), + ), + ], + ids=lambda case: case.description, +) +def test_given_refactor_shell_when_reading_imports_then_only_facts_module_reads_compiled_project( + test_case: ImportBoundaryTestCase, +) -> None: + package_dir: Path = Path(refactor_package.__file__).parent + + importing: list[str] = modules_importing(package_dir=package_dir, name=test_case.imported_name) + + assert tuple(importing) == test_case.expected_modules + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py index 74dab704e9..8cbbdcb454 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py @@ -11,7 +11,7 @@ CompiledProject, CompiledSqlScenario, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -47,7 +47,7 @@ def configure_scenario_runner( monkeypatch.setattr(runner, "run_warehouse_scenarios", lambda **_: 0) -def build_project_with_scenarios(project_dir: Path) -> CompiledProject: +def build_project_with_scenarios(project_dir: Path) -> CompiledProjectView: scenarios: tuple[CompiledSqlScenario, ...] = tuple( CompiledSqlScenario( key=CompiledObjectKey(CompiledResourceType.SQL_SCENARIO, scenario_name), diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_selection.py index 3b0399b474..20d39fed5b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_selection.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_selection.py @@ -6,7 +6,8 @@ from sqlbuild.cli.commands._helpers.scenario_execution.selection import select_scenarios from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlScenario +from sqlbuild.compiler.compile.models import CompiledSqlScenario +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.cli.commands._helpers.scenario_execution._test_types import ( SelectScenariosErrorTestCase, SelectScenariosTestCase, @@ -33,7 +34,7 @@ def test_given_scenario_selectors_when_excluding_then_returns_remaining_scenario tmp_path: Path, ) -> None: project_dir: Path = tmp_path - project: CompiledProject = build_project_with_scenarios(project_dir) + project: CompiledProjectView = build_project_with_scenarios(project_dir) selected: tuple[CompiledSqlScenario, ...] = select_scenarios( project=project, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py index d14691367d..a2d3ebc54b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py @@ -15,7 +15,7 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner.models import PlannerScope from tests.unit.src.sqlbuild.cli.commands.main.build._test_types import ( DeferCloneBoundaryTestCase, @@ -132,7 +132,7 @@ def test_given_disabled_model_in_defer_clone_scope_when_counting_then_only_enabl "a": MODEL_A, "b": MODEL_B, } - models_by_name: dict[str, CompiledModel] = { + models_by_name: dict[str, CompiledModelView] = { name: CompiledModel( key=keys_by_name[name], deps=(), diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index ec3f18b1b1..19c92acac4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -3,7 +3,6 @@ import errno import os from collections.abc import Callable -from dataclasses import replace from pathlib import Path from typing import Any @@ -24,7 +23,13 @@ CompileModelConfig, FunctionArgument, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + FunctionLanguage, +) +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.models import ( AuditPlanEntry, @@ -47,6 +52,7 @@ from tests.unit.src.sqlbuild.cli.commands.main.dag.helpers import ( prepare_python_dag_project, ) +from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( PlanTestChainTestCase, ) @@ -278,7 +284,7 @@ def read_target_files(target_dir: Path, expected_files: dict[str, str]) -> dict[ } -def build_static_target_writer_project() -> CompiledProject: +def build_static_target_writer_project() -> CompiledProjectView: """Build compiled project state for offline compile target writer tests.""" model_key: CompiledObjectKey = CompiledObjectKey( @@ -336,7 +342,7 @@ def build_static_target_writer_project() -> CompiledProject: ) -def build_cached_target_writer_project(*, target_dir: Path) -> CompiledProject: +def build_cached_target_writer_project(*, target_dir: Path) -> CompiledProjectView: """Build a static project containing one cacheable model SQL test.""" compiled_test, project = build_test_and_project( @@ -351,14 +357,14 @@ def build_cached_target_writer_project(*, target_dir: Path) -> CompiledProject: expected_cte_bodies={"orders": "SELECT 2 AS order_id"}, ) ) - return replace( - project, + return project_with( + project=project, compile_cache_dir=target_dir / "cache" / "compiler", sql_tests=(compiled_test,), ) -def build_missing_mock_target_writer_project(*, target_dir: Path) -> CompiledProject: +def build_missing_mock_target_writer_project(*, target_dir: Path) -> CompiledProjectView: """Build a static cached project whose SQL test omits a required source mock.""" compiled_test, project = build_test_and_project( @@ -373,8 +379,8 @@ def build_missing_mock_target_writer_project(*, target_dir: Path) -> CompiledPro expected_cte_bodies={"orders": "SELECT 2 AS order_id"}, ) ) - return replace( - project, + return project_with( + project=project, compile_cache_dir=target_dir / "cache" / "compiler", sql_tests=(compiled_test,), ) @@ -383,7 +389,7 @@ def build_missing_mock_target_writer_project(*, target_dir: Path) -> CompiledPro def build_compile_output_graph(*, model_names: tuple[str, ...]) -> ProjectGraph: """Build a static project graph for compile output formatter tests.""" - models: list[CompiledModel] = [] + models: list[CompiledModelView] = [] upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = {} all_keys: dict[str, CompiledObjectKey] = {} name: str @@ -412,14 +418,14 @@ def build_compile_output_graph(*, model_names: tuple[str, ...]) -> ProjectGraph: upstream_deps[key] = () all_keys[name] = key - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="run-1", effective_target_name="dev", effective_connection={}, effective_vars={}, models=tuple(models), ) - return ProjectGraph( + return project_graph_from_indexes( project=project, upstream_deps=upstream_deps, downstream_deps={key: () for key in upstream_deps}, @@ -439,7 +445,7 @@ def build_linear_compile_output_graph(*, model_count: int) -> ProjectGraph: downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = { model.key: () for model in graph.project.models } - first_model: CompiledModel = graph.project.models[0] + first_model: CompiledModelView = graph.project.models[0] upstream_deps[first_model.key] = () for previous_model, model in zip(graph.project.models, graph.project.models[1:], strict=False): upstream_deps[model.key] = (previous_model.key,) @@ -447,7 +453,7 @@ def build_linear_compile_output_graph(*, model_count: int) -> ProjectGraph: *downstream_deps[previous_model.key], model.key, ) - return ProjectGraph( + return project_graph_from_indexes( project=graph.project, upstream_deps=upstream_deps, downstream_deps=downstream_deps, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index 41a7d6582a..ac3fe4e037 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -14,8 +14,10 @@ from sqlbuild.cli.output.models import ( WrittenTarget, ) -from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey, CompileModelConfig -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileModelConfig +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph @@ -49,8 +51,8 @@ def test_given_large_hook_integer_when_formatting_json_then_preserves_value( test_case: CompileJsonLargeIntegerTestCase, ) -> None: graph: ProjectGraph = build_compile_output_graph(model_names=("orders",)) - model: CompiledModel = replace( - graph.project.models[0], + model: CompiledModelView = replace( + compiled_model_record(graph.project.models[0]), config=CompileModelConfig( values={ "pre_hooks": [ @@ -63,7 +65,7 @@ def test_given_large_hook_integer_when_formatting_json_then_preserves_value( } ), ) - graph = replace(graph, project=replace(graph.project, models=(model,))) + graph = replace(graph, project=project_with(project=graph.project, models=(model,))) payload: dict[str, object] = json.loads( format_compile_json( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 78c05f3c1c..b9d26f4f90 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -17,8 +17,15 @@ from sqlbuild.cli.output.models import ( WrittenTarget, ) -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompiledSqlTest -from sqlbuild.compiler.compile.types import CompiledResourceType, DiagnosticPhase +from sqlbuild.compiler.compile.models import CompiledSqlTest +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + DiagnosticPhase, +) +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( @@ -200,7 +207,7 @@ def test_given_compiled_project_when_writing_static_target_then_expected_files_a test_case: TargetWriterTestCase, tmp_path: Path, ) -> None: - project: CompiledProject = build_static_target_writer_project() + project: CompiledProjectView = build_static_target_writer_project() target_dir: Path = tmp_path / "target" stale_path: Path = target_dir / "compiled" / "models" / "deleted.sql" stale_path.parent.mkdir(parents=True) @@ -225,9 +232,11 @@ def test_given_compiled_project_when_writing_static_target_then_expected_files_a assert not stale_path.exists() assert not (target_dir / "manifest.json").exists() - changed_project: CompiledProject = replace( - project, - models=(replace(project.models[0], query_sql="SELECT 3 AS order_id"),), + changed_project: CompiledProjectView = project_with( + project=project, + models=( + replace(compiled_model_record(project.models[0]), query_sql="SELECT 3 AS order_id"), + ), ) _ = write_static_compile_target( target_dir=target_dir, @@ -253,10 +262,15 @@ def test_given_unicode_sql_when_writing_static_target_twice_then_bytes_remain_un tmp_path: Path, test_case: ExpectedMessageTestCase, ) -> None: - project: CompiledProject = build_static_target_writer_project() - project = replace( - project, - models=(replace(project.models[0], query_sql="SELECT 'café' AS product_name \n\n"),), + project: CompiledProjectView = build_static_target_writer_project() + project = project_with( + project=project, + models=( + replace( + compiled_model_record(project.models[0]), + query_sql="SELECT 'café' AS product_name \n\n", + ), + ), ) target_dir: Path = tmp_path / "target" @@ -291,7 +305,7 @@ def test_given_invalid_utf8_artifact_when_writing_again_then_existing_decode_err tmp_path: Path, test_case: ExpectedMessageTestCase, ) -> None: - project: CompiledProject = build_static_target_writer_project() + project: CompiledProjectView = build_static_target_writer_project() target_dir: Path = tmp_path / "target" _ = write_static_compile_target( target_dir=target_dir, @@ -320,7 +334,7 @@ def test_given_unchanged_test_artifact_when_writing_again_then_skips_test_plan_r test_case: TargetWriterCacheTestCase, ) -> None: target_dir: Path = tmp_path / "target" - project: CompiledProject = build_cached_target_writer_project(target_dir=target_dir) + project: CompiledProjectView = build_cached_target_writer_project(target_dir=target_dir) _ = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), @@ -356,7 +370,7 @@ def test_given_artifact_with_moved_mtime_and_same_bytes_when_writing_then_reuses test_case: TargetWriterCacheTestCase, ) -> None: target_dir: Path = tmp_path / "target" - project: CompiledProject = build_cached_target_writer_project(target_dir=target_dir) + project: CompiledProjectView = build_cached_target_writer_project(target_dir=target_dir) _ = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), @@ -402,7 +416,7 @@ def test_given_test_with_planning_error_when_writing_twice_then_error_is_not_cac test_case: TargetWriterPlanningErrorTestCase, ) -> None: target_dir: Path = tmp_path / "target" - project: CompiledProject = build_missing_mock_target_writer_project(target_dir=target_dir) + project: CompiledProjectView = build_missing_mock_target_writer_project(target_dir=target_dir) first: WrittenTarget = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), @@ -443,7 +457,7 @@ def test_given_changed_model_in_test_closure_when_writing_then_rebuilds_test_art test_case: TargetWriterCacheTestCase, ) -> None: target_dir: Path = tmp_path / "target" - project: CompiledProject = build_cached_target_writer_project(target_dir=target_dir) + project: CompiledProjectView = build_cached_target_writer_project(target_dir=target_dir) _ = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), @@ -451,9 +465,11 @@ def test_given_changed_model_in_test_closure_when_writing_then_rebuilds_test_art ) planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) - changed_project: CompiledProject = replace( - project, - models=(replace(project.models[0], query_sql="SELECT 3 AS order_id"),), + changed_project: CompiledProjectView = project_with( + project=project, + models=( + replace(compiled_model_record(project.models[0]), query_sql="SELECT 3 AS order_id"), + ), ) _ = write_static_compile_target( @@ -476,21 +492,21 @@ def test_given_changed_unrelated_model_when_writing_then_reuses_test_artifact( test_case: TargetWriterCacheTestCase, ) -> None: target_dir: Path = tmp_path / "target" - project: CompiledProject = build_cached_target_writer_project(target_dir=target_dir) + project: CompiledProjectView = build_cached_target_writer_project(target_dir=target_dir) _ = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), project=project, ) - unrelated_model: CompiledModel = replace( - project.models[0], + unrelated_model: CompiledModelView = replace( + compiled_model_record(project.models[0]), key=replace(project.models[0].key, name="unrelated"), name="unrelated", relative_path=Path("models/unrelated.sql"), query_sql="SELECT 9 AS unrelated_id", ) - changed_project: CompiledProject = replace( - project, + changed_project: CompiledProjectView = project_with( + project=project, models=(*project.models, unrelated_model), ) planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) @@ -515,7 +531,7 @@ def test_given_changed_test_sql_when_writing_then_rebuilds_test_artifact( test_case: TargetWriterCacheTestCase, ) -> None: target_dir: Path = tmp_path / "target" - project: CompiledProject = build_cached_target_writer_project(target_dir=target_dir) + project: CompiledProjectView = build_cached_target_writer_project(target_dir=target_dir) _ = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), @@ -530,7 +546,7 @@ def test_given_changed_test_sql_when_writing_then_rebuilds_test_artifact( _ = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), - project=replace(project, sql_tests=(changed_test,)), + project=project_with(project=project, sql_tests=(changed_test,)), ) assert planner_spy.call_count == test_case.expected_builder_calls @@ -551,7 +567,7 @@ def test_given_changed_compile_target_when_writing_then_rebuilds_test_artifact( test_case: TargetWriterCacheTestCase, ) -> None: target_dir: Path = tmp_path / "target" - project: CompiledProject = build_cached_target_writer_project(target_dir=target_dir) + project: CompiledProjectView = build_cached_target_writer_project(target_dir=target_dir) _ = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), @@ -563,7 +579,7 @@ def test_given_changed_compile_target_when_writing_then_rebuilds_test_artifact( _ = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), - project=replace(project, effective_target_schema="changed_schema"), + project=project_with(project=project, effective_target_schema="changed_schema"), ) assert planner_spy.call_count == test_case.expected_builder_calls @@ -580,8 +596,8 @@ def test_given_compile_cache_disabled_when_writing_twice_then_rebuilds_test_arti test_case: TargetWriterCacheTestCase, ) -> None: target_dir: Path = tmp_path / "target" - project: CompiledProject = replace( - build_cached_target_writer_project(target_dir=target_dir), + project: CompiledProjectView = project_with( + project=build_cached_target_writer_project(target_dir=target_dir), compile_cache_dir=None, ) planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) @@ -616,7 +632,7 @@ def test_given_modified_test_artifact_when_writing_then_rebuilds_expected_sql( test_case: TargetWriterCacheTestCase, ) -> None: target_dir: Path = tmp_path / "target" - project: CompiledProject = build_cached_target_writer_project(target_dir=target_dir) + project: CompiledProjectView = build_cached_target_writer_project(target_dir=target_dir) _ = write_static_compile_target( target_dir=target_dir, adapter=DuckDbAdapter(), diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py index fd9b5d9c9c..29e1b644ce 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -2,59 +2,7 @@ from __future__ import annotations -from collections.abc import Callable from dataclasses import dataclass -from pathlib import Path - -import pytest - - -@dataclass(frozen=True) -class ProjectFingerprintTestCase: - """One filesystem change and whether the project fingerprint must report it.""" - - description: str - change: Callable[[Path], None] - expected_unchanged: bool - - -@dataclass(frozen=True) -class DigestCarryForwardTestCase: - """A same-content change whose digest must be verified once and then carried forward.""" - - description: str - change: Callable[[Path], None] - expected_verified: frozenset[str] - expected_carried: frozenset[str] - - -@dataclass(frozen=True) -class RacyRewriteTestCase: - """A rewrite inside one timestamp tick that leaves every stat field unchanged.""" - - description: str - racy: bool - expected_unchanged: bool - - -@dataclass(frozen=True) -class LinkCycleTestCase: - """A directory link that points back into the project.""" - - description: str - link_path: str - expected_present: tuple[str, ...] - expected_descended_prefix: str - - -@dataclass(frozen=True) -class RestampTestCase: - """A stat change to a file stored without a digest, which must miss and queue hashing.""" - - description: str - change: Callable[[Path], None] - expected_unchanged: bool - expected_restamped: frozenset[str] @dataclass(frozen=True) @@ -71,15 +19,6 @@ class ProviderSettingsInputsTestCase: expected_unsupported: bool = False -@dataclass(frozen=True) -class SettingsDigestTestCase: - """One change to the environment or settings files and whether the digest must move.""" - - description: str - change: Callable[[pytest.MonkeyPatch, Path], None] - expected_changed: bool - - @dataclass(frozen=True) class EngineReuseDigestTestCase: """Two engine environments for one invocation and whether their reuse keys may match.""" diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py index 766bd626ac..d8cd78752b 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py @@ -2,8 +2,6 @@ from __future__ import annotations -import os -import time from pathlib import Path from typing import ClassVar @@ -11,28 +9,14 @@ from pydantic import AliasChoices, BaseModel, Field from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict -from sqlbuild.cli.compile_reuse._helpers.project_files import ( - file_digest, - is_racy, - snapshot_project_files, -) -from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( - environment_digest, - invocation_digest, - tracked_environment_names, -) -from sqlbuild.cli.compile_reuse.constants import RACY_WINDOW_NS +from sqlbuild.cli.compile_reuse._helpers.runtime_identity import invocation_identity from sqlbuild.cli.compile_reuse.models import ( CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult, - StoredProjectFile, ) -from sqlbuild.cli.compile_reuse.types import FileStamp from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR -_LATER_SNAPSHOT_NS: int = 10 * RACY_WINDOW_NS - def write_file(path: Path, contents: str) -> None: """Write one file, creating parent folders.""" @@ -41,105 +25,6 @@ def write_file(path: Path, contents: str) -> None: path.write_text(contents, encoding="utf-8") -def write_fingerprint_project(root: Path) -> Path: - """Create a project with links, excluded folders, and presence-only database files.""" - - project_dir: Path = root / "project" - write_file(project_dir / "sqlbuild_project.toml", 'name = "orders"\nadapter = "duckdb"\n') - write_file(project_dir / "models/orders.sql", "SELECT 1 AS order_id\n") - write_file(project_dir / "models/.drafts/returns.sql", "SELECT 2 AS return_id\n") - write_file(project_dir / "macros/currency.py", "def cents() -> str:\n return 'cents'\n") - write_file(project_dir / "macros/__pycache__/currency.cpython-312.pyc", "bytecode") - write_file(project_dir / "target/compiled/models/orders.sql", "SELECT 1 AS order_id\n") - write_file(project_dir / "logs/compile.log", "compile log\n") - write_file(project_dir / ".venv/lib/vendored.py", "VALUE = 1\n") - write_file(project_dir / ".git/HEAD", "ref: refs/heads/main\n") - write_file(project_dir / ".vscode/settings.json", "{}\n") - write_file(project_dir / ".orders_rules/limits.txt", "40\n") - write_file(project_dir / "warehouse.duckdb", "database pages") - write_file(root / "shared/customers.sql", "SELECT 3 AS customer_id\n") - write_file(root / "shared_dir/products.sql", "SELECT 4 AS product_id\n") - (project_dir / "models/customers.sql").symlink_to(root / "shared/customers.sql") - (project_dir / "models/shared").symlink_to(root / "shared_dir", target_is_directory=True) - return project_dir - - -def stored_project_files(*, project_dir: Path, snapshot_ns: int) -> dict[str, StoredProjectFile]: - """Record the project as a stored compile would, with digests for every readable file.""" - - snapshot: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) - return { - relative_path: StoredProjectFile( - stamp=stamp, - digest=file_digest(path=str(project_dir / relative_path)), - racy=is_racy(stamp=stamp, snapshot_ns=snapshot_ns), - ) - for relative_path, stamp in snapshot.items() - } - - -def later_snapshot_ns() -> int: - """Return a snapshot time after which no current file counts as racy.""" - - return time.time_ns() + _LATER_SNAPSHOT_NS - - -def touch_without_change(project_dir: Path) -> None: - """Move a file's mtime forward without changing its content.""" - - path: Path = project_dir / "models/orders.sql" - mtime_ns: int = path.stat().st_mtime_ns + RACY_WINDOW_NS - os.utime(path, ns=(mtime_ns, mtime_ns)) - - -def rewrite_same_size_keeping_mtime(project_dir: Path) -> None: - """Change content without changing size, then restore the previous mtime.""" - - path: Path = project_dir / "models/orders.sql" - mtime_ns: int = path.stat().st_mtime_ns - path.write_text("SELECT 9 AS order_id\n", encoding="utf-8") - os.utime(path, ns=(mtime_ns, mtime_ns)) - - -def edit_link_target(project_dir: Path) -> None: - """Edit the file a project link points to, outside the project.""" - - (project_dir.parent / "shared/customers.sql").write_text( - "SELECT 30 AS customer_id\n", encoding="utf-8" - ) - - -def retarget_link(project_dir: Path) -> None: - """Point a project link at another file with identical content.""" - - link: Path = project_dir / "models/customers.sql" - replacement: Path = project_dir.parent / "shared/customers_copy.sql" - replacement.write_text("SELECT 3 AS customer_id\n", encoding="utf-8") - link.unlink() - link.symlink_to(replacement) - - -def add_file_in_linked_directory(project_dir: Path) -> None: - """Add a file inside a directory the project links to.""" - - write_file(project_dir.parent / "shared_dir/suppliers.sql", "SELECT 5 AS supplier_id\n") - - -def break_link(project_dir: Path) -> None: - """Delete the target of a project link.""" - - (project_dir.parent / "shared/customers.sql").unlink() - - -def stat_only_project_files(*, project_dir: Path) -> dict[str, StoredProjectFile]: - """Record the project as a stored compile does by default: stat stamps without digests.""" - - return { - relative_path: StoredProjectFile(stamp=stamp, digest=None, racy=False) - for relative_path, stamp in snapshot_project_files(project_dir=str(project_dir)).items() - } - - SETTINGS_TOKEN_ENV_VAR: str = "ORDERS_API_TOKEN" SETTINGS_ENV_FILE_NAME: str = "orders_api.env" SETTINGS_SECRETS_DIR_NAME: str = "orders_secrets" @@ -293,16 +178,13 @@ def orders_reuse_request(project_dir: Path) -> CompileReuseRequest: def engine_reuse_key( *, environment: dict[str, str], project_dir: Path, monkeypatch: pytest.MonkeyPatch -) -> tuple[str, str]: - """Return the invocation and environment digests computed under one engine environment.""" +) -> bytes: + """Return the encoded invocation identity under one engine environment.""" monkeypatch.delenv(COMPILER_ENGINE_ENV_VAR, raising=False) monkeypatch.delenv(STAGE_CAPTURE_DIR_ENV_VAR, raising=False) for name, value in environment.items(): monkeypatch.setenv(name, value) - return ( - invocation_digest( - request=orders_reuse_request(project_dir), project_dir=str(project_dir), use_color=False - ), - environment_digest(names=tracked_environment_names(template_names=())), + return invocation_identity( + request=orders_reuse_request(project_dir), project_dir=str(project_dir), use_color=False ) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py index 76965985e1..4233d4d975 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py @@ -9,7 +9,7 @@ from sqlbuild.cli.compile_reuse._helpers.attempt import attempt_reuse from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt -from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( EngineReuseDigestTestCase, EngineReuseStoreTestCase, @@ -48,22 +48,16 @@ second_environment={COMPILER_ENGINE_ENV_VAR: "native"}, expected_same_digest=True, ), - EngineReuseDigestTestCase( - description="stage_capture_directory_is_not_a_compile_input", - first_environment={}, - second_environment={STAGE_CAPTURE_DIR_ENV_VAR: "/tmp/captures"}, - expected_same_digest=True, - ), ], ids=lambda case: case.description, ) def test_given_same_invocation_when_engine_environment_changes_then_reuse_key_follows_engine( test_case: EngineReuseDigestTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - first: tuple[str, str] = engine_reuse_key( + first: bytes = engine_reuse_key( environment=test_case.first_environment, project_dir=tmp_path, monkeypatch=monkeypatch ) - second: tuple[str, str] = engine_reuse_key( + second: bytes = engine_reuse_key( environment=test_case.second_environment, project_dir=tmp_path, monkeypatch=monkeypatch ) @@ -76,12 +70,12 @@ def test_given_same_invocation_when_engine_environment_changes_then_reuse_key_fo EngineReuseStoreTestCase( description="native_preview", engine="native-preview", - expected_entry_directory="target/cache/compiler-native-preview-v1/project-reuse-v1", + expected_entry_directory="target/cache/compiler-native-preview-v1/project-reuse-v2", ), EngineReuseStoreTestCase( description="native", engine="native", - expected_entry_directory="target/cache/compiler-native-v1/project-reuse-v1", + expected_entry_directory="target/cache/compiler-native-v1/project-reuse-v2", ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py deleted file mode 100644 index 1de9e2e1a5..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py +++ /dev/null @@ -1,286 +0,0 @@ -"""Project input fingerprint edge cases for whole-project compile reuse.""" - -from __future__ import annotations - -import time -from dataclasses import replace -from pathlib import Path - -import pytest - -from sqlbuild.cli.compile_reuse._helpers.project_files import ( - carried_forward_digests, - compare_project_files, - restamped_paths, - snapshot_project_files, -) -from sqlbuild.cli.compile_reuse.models import ProjectFilesComparison, StoredProjectFile -from sqlbuild.cli.compile_reuse.types import FileStamp -from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( - DigestCarryForwardTestCase, - LinkCycleTestCase, - ProjectFingerprintTestCase, - RacyRewriteTestCase, - RestampTestCase, -) -from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import ( - add_file_in_linked_directory, - break_link, - edit_link_target, - later_snapshot_ns, - retarget_link, - rewrite_same_size_keeping_mtime, - stat_only_project_files, - stored_project_files, - touch_without_change, - write_file, - write_fingerprint_project, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ProjectFingerprintTestCase( - description="no_change", change=lambda _root: None, expected_unchanged=True - ), - ProjectFingerprintTestCase( - description="mtime_only_change_same_content", - change=touch_without_change, - expected_unchanged=True, - ), - ProjectFingerprintTestCase( - description="same_size_and_mtime_new_content", - change=rewrite_same_size_keeping_mtime, - expected_unchanged=False, - ), - ProjectFingerprintTestCase( - description="file_added", - change=lambda root: write_file(root / "models/returns.sql", "SELECT 1\n"), - expected_unchanged=False, - ), - ProjectFingerprintTestCase( - description="file_removed", - change=lambda root: (root / "macros/currency.py").unlink(), - expected_unchanged=False, - ), - ProjectFingerprintTestCase( - description="empty_directory_added", - change=lambda root: (root / "models/_sqlbuild").mkdir(), - expected_unchanged=False, - ), - ProjectFingerprintTestCase( - description="nested_hidden_directory_edit", - change=lambda root: write_file(root / "models/.drafts/returns.sql", "SELECT 22\n"), - expected_unchanged=False, - ), - ProjectFingerprintTestCase( - description="file_link_target_edit", change=edit_link_target, expected_unchanged=False - ), - ProjectFingerprintTestCase( - description="file_link_retarget", change=retarget_link, expected_unchanged=False - ), - ProjectFingerprintTestCase( - description="file_added_in_linked_directory", - change=add_file_in_linked_directory, - expected_unchanged=False, - ), - ProjectFingerprintTestCase( - description="file_link_broken", change=break_link, expected_unchanged=False - ), - ProjectFingerprintTestCase( - description="database_file_removed", - change=lambda root: (root / "warehouse.duckdb").unlink(), - expected_unchanged=False, - ), - ProjectFingerprintTestCase( - description="database_file_content_change", - change=lambda root: write_file(root / "warehouse.duckdb", "rewritten database pages"), - expected_unchanged=True, - ), - ProjectFingerprintTestCase( - description="target_directory_change", - change=lambda root: write_file( - root / "target/compiled/models/orders.sql", "SELECT 2\n" - ), - expected_unchanged=True, - ), - ProjectFingerprintTestCase( - description="compute_log_change", - change=lambda root: write_file(root / "logs/compile-2.log", "next compile log\n"), - expected_unchanged=True, - ), - ProjectFingerprintTestCase( - description="bytecode_cache_change", - change=lambda root: write_file(root / "macros/__pycache__/currency.pyc", "bytecode 2"), - expected_unchanged=True, - ), - ProjectFingerprintTestCase( - description="root_hidden_directory_change", - change=lambda root: write_file(root / ".venv/lib/vendored.py", "VALUE = 2\n"), - expected_unchanged=True, - ), - ProjectFingerprintTestCase( - description="root_editor_settings_change", - change=lambda root: write_file(root / ".vscode/settings.json", '{"tabs": 2}\n'), - expected_unchanged=True, - ), - ProjectFingerprintTestCase( - description="root_hidden_project_directory_change", - change=lambda root: write_file(root / ".orders_rules/limits.txt", "400\n"), - expected_unchanged=False, - ), - ProjectFingerprintTestCase( - description="version_control_metadata_change", - change=lambda root: write_file(root / ".git/HEAD", "ref: refs/heads/feature\n"), - expected_unchanged=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_fingerprint_when_project_changes_then_change_is_detected_exactly( - test_case: ProjectFingerprintTestCase, tmp_path: Path -) -> None: - project_dir: Path = write_fingerprint_project(tmp_path) - stored: dict[str, StoredProjectFile] = stored_project_files( - project_dir=project_dir, snapshot_ns=later_snapshot_ns() - ) - - test_case.change(project_dir) - comparison: ProjectFilesComparison = compare_project_files( - project_dir=str(project_dir), - stored=stored, - current=snapshot_project_files(project_dir=str(project_dir)), - ) - - assert comparison.unchanged is test_case.expected_unchanged - - -@pytest.mark.parametrize( - "test_case", - [ - DigestCarryForwardTestCase( - description="touched_model", - change=touch_without_change, - expected_verified=frozenset({"models/orders.sql"}), - expected_carried=frozenset({"models/orders.sql", "macros/currency.py"}), - ) - ], - ids=lambda case: case.description, -) -def test_given_touched_file_with_same_content_when_checking_then_its_digest_carries_forward( - test_case: DigestCarryForwardTestCase, tmp_path: Path -) -> None: - project_dir: Path = write_fingerprint_project(tmp_path) - stored: dict[str, StoredProjectFile] = stored_project_files( - project_dir=project_dir, snapshot_ns=later_snapshot_ns() - ) - test_case.change(project_dir) - current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) - - comparison: ProjectFilesComparison = compare_project_files( - project_dir=str(project_dir), stored=stored, current=current - ) - digests: dict[str, str] = carried_forward_digests( - stored=stored, current=current, verified=comparison.verified - ) - - assert comparison.unchanged - assert frozenset(comparison.verified) == test_case.expected_verified - assert test_case.expected_carried <= frozenset(digests) - assert {path: digests[path] for path in test_case.expected_carried} == { - path: stored[path].digest for path in test_case.expected_carried - } - - -@pytest.mark.parametrize( - "test_case", - [ - RestampTestCase( - description="touched_file_without_digest", - change=touch_without_change, - expected_unchanged=False, - expected_restamped=frozenset({"models/orders.sql"}), - ), - RestampTestCase( - description="unchanged_file_without_digest", - change=lambda _root: None, - expected_unchanged=True, - expected_restamped=frozenset(), - ), - ], - ids=lambda case: case.description, -) -def test_given_stat_only_stored_files_when_a_stamp_moves_then_it_misses_and_is_queued_for_hashing( - test_case: RestampTestCase, tmp_path: Path -) -> None: - project_dir: Path = write_fingerprint_project(tmp_path) - stored: dict[str, StoredProjectFile] = stat_only_project_files(project_dir=project_dir) - - test_case.change(project_dir) - current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) - comparison: ProjectFilesComparison = compare_project_files( - project_dir=str(project_dir), stored=stored, current=current - ) - - assert comparison.unchanged is test_case.expected_unchanged - assert comparison.verified == {} - assert restamped_paths(stored=stored, current=current) == test_case.expected_restamped - - -@pytest.mark.parametrize( - "test_case", - [ - RacyRewriteTestCase(description="racy_stamp_rehashed", racy=True, expected_unchanged=False), - RacyRewriteTestCase( - description="settled_stamp_trusted", racy=False, expected_unchanged=True - ), - ], - ids=lambda case: case.description, -) -def test_given_rewrite_within_one_timestamp_tick_when_checking_then_only_racy_stamps_rehash( - test_case: RacyRewriteTestCase, tmp_path: Path -) -> None: - project_dir: Path = write_fingerprint_project(tmp_path) - stored: dict[str, StoredProjectFile] = stored_project_files( - project_dir=project_dir, snapshot_ns=time.time_ns() - ) - current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) - (project_dir / "models/orders.sql").write_text("SELECT 9 AS order_id\n", encoding="utf-8") - stored["models/orders.sql"] = replace( - stored["models/orders.sql"], stamp=current["models/orders.sql"], racy=test_case.racy - ) - - comparison: ProjectFilesComparison = compare_project_files( - project_dir=str(project_dir), stored=stored, current=current - ) - - assert comparison.unchanged is test_case.expected_unchanged - - -@pytest.mark.parametrize( - "test_case", - [ - LinkCycleTestCase( - description="link_to_project_root", - link_path="models/loop", - expected_present=("models/loop", "models/orders.sql"), - expected_descended_prefix="models/loop/", - ) - ], - ids=lambda case: case.description, -) -def test_given_directory_link_cycle_when_snapshotting_then_walk_terminates( - test_case: LinkCycleTestCase, tmp_path: Path -) -> None: - project_dir: Path = write_fingerprint_project(tmp_path) - (project_dir / test_case.link_path).symlink_to(project_dir, target_is_directory=True) - - snapshot: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) - - assert frozenset(test_case.expected_present) <= frozenset(snapshot) - assert not any(path.startswith(test_case.expected_descended_prefix) for path in snapshot) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_provider_settings.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_provider_settings.py index eb01b8c105..ea61bf3fb3 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_provider_settings.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_provider_settings.py @@ -2,21 +2,16 @@ from __future__ import annotations -from pathlib import Path - import pytest from sqlbuild.cli.compile_reuse._helpers.provider_settings import provider_settings_inputs -from sqlbuild.cli.compile_reuse._helpers.runtime_identity import settings_inputs_digest -from sqlbuild.cli.compile_reuse.models import SettingsEnvironmentInputs, SettingsInputsResult +from sqlbuild.cli.compile_reuse.models import SettingsInputsResult from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( ProviderSettingsInputsTestCase, - SettingsDigestTestCase, ) from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import ( SETTINGS_ENV_FILE_NAME, SETTINGS_SECRETS_DIR_NAME, - SETTINGS_TOKEN_ENV_VAR, AliasedOrdersSettings, CaseSensitiveOrdersSettings, CommandLineOrdersSettings, @@ -26,9 +21,6 @@ OrdersApiSettings, PrefixedOrdersSettings, described_settings_inputs, - file_settings_inputs, - set_token, - write_settings_file, ) @@ -119,58 +111,5 @@ def test_given_settings_with_unlisted_sources_when_enumerating_then_reuse_is_ref ) -@pytest.mark.parametrize( - "test_case", - [ - SettingsDigestTestCase( - description="unrelated_variable", - change=lambda mp, _root: mp.setenv("OTHER", "1"), - expected_changed=False, - ), - SettingsDigestTestCase( - description="token_unset", - change=lambda mp, _root: mp.delenv(SETTINGS_TOKEN_ENV_VAR), - expected_changed=True, - ), - SettingsDigestTestCase( - description="token_changed", - change=lambda mp, _root: set_token(mp, "beta"), - expected_changed=True, - ), - SettingsDigestTestCase( - description="token_set_in_other_case", - change=lambda mp, _root: mp.setenv(SETTINGS_TOKEN_ENV_VAR.lower(), "gamma"), - expected_changed=True, - ), - SettingsDigestTestCase( - description="env_file_changed", - change=lambda _mp, root: write_settings_file( - root, SETTINGS_ENV_FILE_NAME, "ORDERS_API_TOKEN=delta\n" - ), - expected_changed=True, - ), - SettingsDigestTestCase( - description="secret_added", - change=lambda _mp, root: write_settings_file( - root, f"{SETTINGS_SECRETS_DIR_NAME}/orders_api_token", "epsilon" - ), - expected_changed=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_settings_inputs_when_their_values_change_then_the_digest_moves( - test_case: SettingsDigestTestCase, monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - set_token(monkeypatch, "alpha") - write_settings_file(tmp_path, SETTINGS_ENV_FILE_NAME, "ORDERS_API_TOKEN=alpha\n") - inputs: tuple[SettingsEnvironmentInputs, ...] = (file_settings_inputs(root=tmp_path),) - before: str = settings_inputs_digest(inputs=inputs) - - test_case.change(monkeypatch, tmp_path) - - assert (settings_inputs_digest(inputs=inputs) != before) is test_case.expected_changed - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 2419372f0b..354ec57e68 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -6,12 +6,11 @@ from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledObjectKey, - CompileProjectInputs, CompileSqlReference, ExpansionSpan, InferredColumn, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, SqlTestMode +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompileInputsView from sqlbuild.compiler.lineage.types import ( InferredNullability, ) @@ -20,13 +19,6 @@ from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention, SourceLocation from sqlbuild.sql_values.models import SqlValue -from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import RequiredDescriptionInputs - - -@dataclass(frozen=True) -class UnexpectedAnalysisFailureTestCase: - description: str - expected_error: str @dataclass(frozen=True) @@ -36,15 +28,6 @@ class SqlReferenceExtractionTestCase: expected_references: tuple[tuple[SqlReferenceKind, str, str | None], ...] -@dataclass(frozen=True) -class DialectCteScanTestCase: - description: str - syntax: SqlLexicalSyntax - sql: str - expected_cte_names: tuple[str, ...] - expected_assertion_targets: tuple[str, ...] - - @dataclass(frozen=True) class DialectSqlScanTestCase: description: str @@ -82,30 +65,6 @@ class AuditFactoryAttachmentTestCase: expected_warning_code: str | None = None -@dataclass(frozen=True) -class NativeSqlTestExtractionParityTestCase: - description: str - sql: str - mode: SqlTestMode - expected_matches: bool = True - - -@dataclass(frozen=True) -class QualifiedReferenceAnalysisTestCase: - description: str - query_sql: str - expected_matches: bool = True - - -@dataclass(frozen=True) -class NativeTypeInferenceModeTestCase: - description: str - query_sql: str - column_types: dict[str, str] - expected_fast_types: tuple[str | None, ...] - expected_rich_types: tuple[str | None, ...] - - @dataclass(frozen=True) class WatermarkLimitValidationTestCase: description: str @@ -263,7 +222,7 @@ class ScopedDeclarationErrorTestCase: class ScopedDeclarationSurfaceTestCase: description: str files: dict[str, str] - result: Callable[[CompileProjectInputs], str | None] + result: Callable[[CompileInputsView], str | None] expected_sql: str @@ -374,87 +333,6 @@ class DynamicModelSchemaTestCase: expected_fixed_columns: tuple[str, ...] = () -@dataclass(frozen=True) -class ExtractSqlTestCtesTestCase: - description: str - sql: str - expected_authored_cte_names: tuple[str, ...] - expected_mock_model_names: tuple[str, ...] - expected_mock_source_names: tuple[str, ...] - expected_expected_model_names: tuple[str, ...] - expected_mock_seed_names: tuple[str, ...] = () - expected_mock_dbt_ref_names: tuple[str, ...] = () - expected_mock_table_function_names: tuple[str, ...] = () - expected_assertion_names: tuple[str, ...] = () - expected_macro_mocks: dict[str, str] = field(default_factory=dict) - mode: SqlTestMode = SqlTestMode.MODEL - expected_macro_actual_cte_name: str | None = None - expected_macro_expected_cte_name: str | None = None - - -@dataclass(frozen=True) -class ExtractSqlTestCtesErrorTestCase: - description: str - sql: str - expected_error_fragment: str - mode: SqlTestMode = SqlTestMode.MODEL - - -@dataclass(frozen=True) -class ExtractSqlScenarioCtesTestCase: - description: str - sql: str - expected_authored_cte_names: tuple[str, ...] - expected_source_fixture_names: tuple[str, ...] - expected_ref_fixture_names: tuple[str, ...] - expected_expected_model_names: tuple[str, ...] - expected_assertion_names: tuple[str, ...] - expected_seed_fixture_names: tuple[str, ...] = () - expected_dbt_ref_fixture_names: tuple[str, ...] = () - - -@dataclass(frozen=True) -class ExtractSqlScenarioCtesErrorTestCase: - description: str - sql: str - expected_error_fragment: str - - -@dataclass(frozen=True) -class BuildScenarioInputsTestCase: - description: str - sql_body: str - effective_vars: dict[str, object] - expected_source_fixture_names: tuple[str, ...] - expected_ref_fixture_names: tuple[str, ...] - expected_seed_fixture_names: tuple[str, ...] - expected_expected_model_names: tuple[str, ...] - expected_assertion_names: tuple[str, ...] - expected_sql_fragment: str - expected_dbt_ref_fixture_names: tuple[str, ...] = () - - -@dataclass(frozen=True) -class BuildScenarioInputsErrorTestCase: - description: str - sql_body: str - expected_error_fragment: str - - -@dataclass(frozen=True) -class ExtractSqlAnalysisExpectedBranchesTestCase: - description: str - sql: str - expected_branch_column_names: tuple[tuple[str, ...], ...] | None - - -@dataclass(frozen=True) -class ExtractSqlAnalysisExpectedBranchesErrorTestCase: - description: str - sql: str - expected_error_fragment: str - - @dataclass(frozen=True) class AssembleCompiledProjectTestCase: description: str @@ -565,16 +443,6 @@ class ValidateSqlSyntaxTestCase: dialect: str | None = None -@dataclass(frozen=True) -class PreservedLogicalNamespaceTestCase: - description: str - logical_database: str | None - logical_schema: str | None - target_database: str | None - target_schema: str | None - expected_error_fragment: str | None - - @dataclass(frozen=True) class ValidateHookSqlSyntaxTestCase: description: str @@ -696,48 +564,6 @@ class SnapshotConfigErrorTestCase: expected_error_code: str = "P001" -@dataclass(frozen=True) -class ResolveAuditSeverityTestCase: - description: str - instance_severity: str | None - default_severity: str | None - expected_severity: str | None = None - expected_error_fragment: str | None = None - - -@dataclass(frozen=True) -class ResolveAuditRunScopeTestCase: - description: str - instance_run_scope: str | None - default_run_scope: str | None - expected_run_scope: str - - -@dataclass(frozen=True) -class ResolveAuditRunScopeErrorTestCase: - description: str - instance_run_scope: str | None - default_run_scope: str | None - expected_error_fragment: str - - -@dataclass(frozen=True) -class ValidateModelAttachedAuditRefsTestCase: - description: str - references: tuple[CompileSqlReference, ...] - attached_target_kind: str - attached_target_name: str - expected_valid: bool = True - expected_error_fragment: str | None = None - - -@dataclass(frozen=True) -class ValidateAuditRefsErrorTestCase: - description: str - references: tuple[CompileSqlReference, ...] - expected_error_fragment: str - - @dataclass(frozen=True) class CustomMaterializationConfigErrorTestCase: description: str @@ -852,50 +678,6 @@ class ExpectedCountTestCase: expected_count: int -@dataclass(frozen=True) -class ClassifyDirectLogicSqlTestCtesErrorTestCase: - description: str - mode: SqlTestMode - ctes: tuple[tuple[str, str], ...] - expected_message: str - - -@dataclass(frozen=True) -class CteScannerMessageTestCase: - description: str - sql: str - expected_message: str - - -@dataclass(frozen=True) -class CompleteCeremonialSelectTestCase: - description: str - sql: str - expected_sql: str - - -@dataclass(frozen=True) -class SetOperationExpectedKind: - """One SQL-test kind whose expected CTE accepts set operations.""" - - description: str - mode: SqlTestMode - support_ctes: tuple[tuple[str, str], ...] - expected_cte_name: str - payload_type: str - - -@dataclass(frozen=True) -class SetOperationExpectedTestCase: - """One expected-CTE set operation in one SQL-test kind.""" - - description: str - kind: SetOperationExpectedKind - sql: str - expected_payload_type: str = "" - expected_error_template: str = "" - - @dataclass(frozen=True) class NamedDeclarationAcceptedTestCase: """A valid audit, schema, or hook layout.""" @@ -1054,24 +836,6 @@ class RebindDeclarationScopeTestCase: private_macro_source: str | None = None -@dataclass(frozen=True) -class QualifiedReferenceScanTestCase: - description: str - query_sql: str - reference_names: tuple[str, ...] - expected_names: frozenset[str] - - -@dataclass(frozen=True) -class RequiredDescriptionUnitCase: - """One resource kind checked by the required-description compile rule.""" - - description: str - build: Callable[[str | None], RequiredDescriptionInputs] - expected_message: str - expected_location: tuple[str, int, int] - - @dataclass(frozen=True) class DescriptionInputClassificationCase: """Discovered input fields split between required and exempt description kinds.""" @@ -1082,26 +846,6 @@ class DescriptionInputClassificationCase: expected_fields: frozenset[str] -@dataclass(frozen=True) -class RequiredDescriptionLocationCase: - """Where a P010 finding points when comments or docstrings precede the declaration.""" - - description: str - build: Callable[[], RequiredDescriptionInputs] - expected_location: tuple[str, int, int] - - -@dataclass(frozen=True) -class CursorModelWithoutInputsCase: - """One model header and query checked by the cursor-input compile check (P011).""" - - description: str - config: dict[str, object] - reference_kinds: tuple[str, ...] - expected_codes: tuple[str, ...] - expected_locations: tuple[tuple[str, int, int, int], ...] = () - - @dataclass(frozen=True) class CollectedReferenceCallSyntaxTestCase: description: str @@ -1110,14 +854,6 @@ class CollectedReferenceCallSyntaxTestCase: expected_diagnostics: tuple[tuple[str, str, Path, SourceLocation | None], ...] -@dataclass(frozen=True) -class AuthoredOffsetTestCase: - description: str - contents: str - marker: str - expected_line_column: tuple[int, int] - - @dataclass(frozen=True) class UndecodableSecretTestCase: """An env or var value holding a secret-like prefix and invalid text after it.""" diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py index b662a814da..2eccb09ea6 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py @@ -5,7 +5,11 @@ from sqlbuild.compiler.analysis_session._helpers.dynamic_contracts import ( native_dynamic_contracts, ) +from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import ( + dynamic_contract_proof as contract_proof, +) from sqlbuild.compiler.analysis_session.models import NativePivotTables +from sqlbuild.compiler.compile.models import DynamicColumnContractProof from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.compile._helpers.analysis._test_types import ( DynamicPivotAnalysisTestCase, @@ -147,7 +151,7 @@ def test_given_dynamic_family_when_analyzing_output_then_requires_supported_closed_shape( test_case: DynamicPivotAnalysisTestCase, ) -> None: - (proof,) = native_dynamic_contracts( + (row,) = native_dynamic_contracts( session=None, tables=NativePivotTables( dialect=test_case.dialect, @@ -190,6 +194,7 @@ def test_given_dynamic_family_when_analyzing_output_then_requires_supported_clos ), ), ) + proof: DynamicColumnContractProof | None = contract_proof(row) assert proof is not None assert proof.output_proven is test_case.expected_proven diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_set_operations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_set_operations.py index 1d8757d007..947e57abaa 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_set_operations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_set_operations.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.compile._helpers.analysis.syntax_checks import names_set_operation +import sqlbuild._native as _native from tests.unit.src.sqlbuild.compiler.compile._helpers.analysis._test_types import ( SetOperationSearchTestCase, ) @@ -55,7 +55,7 @@ def test_given_generated_sql_when_searching_set_operations_then_matches_python_s ] expected: list[bool] = [_PYTHON_SEARCH.search(sql) is not None for sql in sqls] - assert [names_set_operation(sql) for sql in sqls] == expected + assert [_native._oracle_names_set_operation(sql) for sql in sqls] == expected assert sum(expected) >= test_case.expected_minimum_matches diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index d6cec06921..dab596add8 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -2,14 +2,13 @@ import sys import unicodedata -from collections.abc import Callable +from dataclasses import dataclass from pathlib import Path from typing import Any import pytest import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core from sqlbuild.compiler.compile._helpers.attachment.model_config import ( build_native_model_config, native_model_config_session, @@ -18,11 +17,11 @@ from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileModelConfig, - CompileProjectInputs, ModelResourceNames, NativeModelConfigInputs, NativeModelConfigSession, ) +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.spec.contracts.models import ( @@ -43,19 +42,16 @@ def compile_with_config_build_count( *, project_dir: Path, monkeypatch: pytest.MonkeyPatch, -) -> tuple[CompileProjectInputs, int]: +) -> tuple[CompileInputsView, int]: """Compile a project while counting its native model config builds.""" build_counts: list[int] = [0] - original_build_model_config: Callable[..., CompileModelConfig] = ( - attachment_core.build_native_model_config + native_loop_type: type = _native.NativeModelLoop + monkeypatch.setattr( + _native, + "NativeModelLoop", + lambda *arguments: _ConfigCountingLoop(native_loop_type(*arguments), build_counts), ) - - def counting_build_model_config(**kwargs: Any) -> CompileModelConfig: - build_counts[0] += 1 - return original_build_model_config(**kwargs) - - monkeypatch.setattr(attachment_core, "build_native_model_config", counting_build_model_config) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) return ( build_compile_inputs( @@ -66,7 +62,20 @@ def counting_build_model_config(**kwargs: Any) -> CompileModelConfig: ) -def compile_input_schema_columns(inputs: CompileProjectInputs) -> tuple[SchemaColumn, ...]: +@dataclass(frozen=True) +class _ConfigCountingLoop: + """The native model loop, adding up the model config builds it runs.""" + + native_loop: Any + build_counts: list[int] + + def run(self, *arguments: object) -> tuple[list[object], tuple[int, int, int]]: + inputs, counts = self.native_loop.run(*arguments) + self.build_counts[0] += counts[1] + return inputs, counts + + +def compile_input_schema_columns(inputs: CompileInputsView) -> tuple[SchemaColumn, ...]: """Flatten attached schema columns in model input order.""" columns: list[SchemaColumn] = [] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index d87ed36445..a094251b3a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -1,6 +1,5 @@ import itertools from collections.abc import Callable -from dataclasses import dataclass, replace from pathlib import Path from types import MappingProxyType from typing import cast @@ -16,18 +15,13 @@ infer_native_query_columns, ) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( collect_compile_diagnostics, ) -from sqlbuild.compiler.compile._helpers.diagnostics.descriptions import ( - missing_description_diagnostics, -) from sqlbuild.compiler.compile._helpers.render.macros import ( expand_sql_macros, load_project_macros, ) -from sqlbuild.compiler.compile._helpers.sql_tests.native import extract_unexpanded_sql_test from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( @@ -35,21 +29,12 @@ CompiledDirectLogicSqlTestPayload, CompiledLineageColumnFact, CompiledLineageSourceFact, - CompiledModel, CompiledModelSqlTestPayload, CompiledObjectKey, - CompiledProject, CompiledRelationLocation, CompiledSqlTest, - CompileModelConfig, - CompileModelInput, - CompileProjectInputs, CompilerDiagnostic, - CompileSeedInput, - CompileSourceInput, - CompileSqlFunctionInput, CompileSqlReference, - CompileSqlTestCtes, DeclarationExpansionContext, DeclarationResolutionContext, DeclarationRuntimeProjection, @@ -57,23 +42,19 @@ LoadedMacro, MacroContext, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage, SqlTestMode +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompileInputsView, + ModelInput, +) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( - DiscoveredLoaderFunction, DiscoveredMacroFile, DiscoveredProjectInputs, - DiscoveredProvider, - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSourceFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, - DiscoveredSqlModelFile, - DiscoveredSqlScenarioFile, - DiscoveredTaskFunction, ) -from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import build_lineage_upstream_deps +from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnTransformKind, @@ -94,19 +75,8 @@ ResourceRecord, ScopeIndex, ScopeLookup, - VisibilityRecord, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, ScopeKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.providers import Provider -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - SchemaModelEntry, - SchemaSeedEntry, - SourceEntry, - SourceLocation, -) from sqlbuild.sql_values.types import CollectionRendering DUCKDB_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( @@ -128,26 +98,6 @@ ) -def visible_declarations_without_runtime_values( - *, enum_visibility: dict[str, tuple[VisibilityRecord, ...]] | None = None -) -> attachment_core._VisibleModelDeclarations: - """Build declaration visibility facts without runtime declaration values.""" - - return attachment_core._VisibleModelDeclarations( - local_enums={}, - local_constants={}, - enums={}, - constants={}, - inaccessible_enums={}, - inaccessible_constants={}, - enum_visibility=enum_visibility or {}, - constant_visibility={}, - macros={}, - macro_records={}, - inaccessible_macros={}, - ) - - def direct_orders_lineage(*column_names: str) -> tuple[CompiledLineageColumnFact, ...]: return tuple( CompiledLineageColumnFact( @@ -232,23 +182,23 @@ def build_loaded_macros(tmp_path: Path, macro_file_contents: str) -> dict[str, L ) -def compile_first_model(*, project_dir: Path) -> CompiledModel: +def compile_first_model(*, project_dir: Path) -> CompiledModelView: """Compile a fixture project and return its first model.""" discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run", ) - compiled_project: CompiledProject = assemble_project( + compiled_project: CompiledProjectView = assemble_project( inputs=compile_inputs, skip_column_inference=True, ) return compiled_project.models[0] -def compile_project_inputs(*, project_dir: Path) -> CompileProjectInputs: +def compile_project_inputs(*, project_dir: Path) -> CompileInputsView: """Compile a discovered DuckDB fixture through the input attachment boundary.""" return build_compile_inputs( @@ -308,7 +258,7 @@ def model_header(*, key: str, value: str) -> str: return f"MODEL (description 'Test model check.', {key} {value});\nSELECT 1 AS order_id" -def compile_and_assemble(*, project_dir: Path) -> CompiledProject: +def compile_and_assemble(*, project_dir: Path) -> CompiledProjectView: """Attach and assemble a DuckDB fixture so scope placement is validated.""" return assemble_project( @@ -328,7 +278,7 @@ def gate_model(*, sql: str, header: str = "MODEL (description 'Test model.');") return f"{header}\n{sql}" -def execution_edge_names(*, project: CompiledProject) -> frozenset[tuple[str, str]]: +def execution_edge_names(*, project: CompiledProjectView) -> frozenset[tuple[str, str]]: """Return (node, upstream) name pairs of the project's execution graph.""" edges: set[tuple[str, str]] = set() @@ -345,13 +295,13 @@ def inline_sql_hook_header(sql: str) -> str: return f"MODEL (description 'Test model.', pre_hooks [inline_sql('{sql}')]);" -def lineage_edge_names(*, project: CompiledProject) -> frozenset[tuple[str, str]]: +def lineage_edge_names(*, project: CompiledProjectView) -> frozenset[tuple[str, str]]: """Return (node, upstream) name pairs of the project's lineage graph used for selection.""" edges: set[tuple[str, str]] = set() key: CompiledObjectKey deps: tuple[CompiledObjectKey, ...] - for key, deps in build_lineage_upstream_deps(project).items(): + for key, deps in project_lineage_views(project).upstream_deps.items(): edges.update((key.name, dep.name) for dep in deps) return frozenset(edges) @@ -464,7 +414,7 @@ def collect_typed_macro_violations( return violations.diagnostics -def render_compile_diagnostics(*, project: CompiledProject) -> str: +def render_compile_diagnostics(*, project: CompiledProjectView) -> str: """Render every project diagnostic with its code, message, location, and help.""" return "\n".join( @@ -503,325 +453,10 @@ def write_scoped_macro_orders_project(project_dir: Path) -> None: ) -REQUIRED_DESCRIPTION_BASE_INPUTS: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig(name="orders", adapter="duckdb"), - local_config=LocalConfig(), -) - - -@dataclass(frozen=True) -class RequiredDescriptionInputs: - """Compile inputs passed to the required-description check.""" - - discovered_inputs: DiscoveredProjectInputs = REQUIRED_DESCRIPTION_BASE_INPUTS - model_inputs: tuple[CompileModelInput, ...] = () - seed_inputs: tuple[CompileSeedInput, ...] = () - source_inputs: tuple[CompileSourceInput, ...] = () - function_inputs: tuple[CompileSqlFunctionInput, ...] = () - - -class OrdersApi(Provider): - """Client for the orders API.""" - - -class UndescribedApi(Provider): - pass - - -_PROVIDER_CLASS_BY_UNDESCRIBED: dict[bool, type[Provider]] = { - False: OrdersApi, - True: UndescribedApi, -} - - -def refresh_exports_task(ctx: object) -> None: - del ctx - - -def documented_returns_loader(ctx: object) -> None: - """Returned orders from the support desk.""" - - del ctx - - -def model_description_inputs( - description: str | None, - *, - contents: str = "-- totals\nMODEL (materialized table);\nSELECT 1 AS id\n", -) -> RequiredDescriptionInputs: - return RequiredDescriptionInputs( - model_inputs=( - CompileModelInput( - model_file=DiscoveredSqlModelFile( - file_path=Path("/project/models/order_totals.sql"), - relative_path=Path("models/order_totals.sql"), - contents=contents, - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="SELECT 1 AS id", - ), - schema_entry=SchemaModelEntry(name="order_totals", description=description), - ), - ) - ) - - -def scenario_description_inputs( - description: str | None, *, contents: str = "\nSCENARIO ();\nSELECT 1\n" -) -> RequiredDescriptionInputs: - return RequiredDescriptionInputs( - discovered_inputs=replace( - REQUIRED_DESCRIPTION_BASE_INPUTS, - scenario_files=( - DiscoveredSqlScenarioFile( - file_path=Path("/project/tests/scenarios/orders__paid.sql"), - relative_path=Path("tests/scenarios/orders__paid.sql"), - contents=contents, - header_values={"description": description}, - sql_body="SELECT 1", - name="orders__paid", - ), - ), - ) - ) - - -def seed_description_inputs(description: str | None) -> RequiredDescriptionInputs: - contents: str = "seeds:\n - name: product_types\n" - return RequiredDescriptionInputs( - seed_inputs=( - CompileSeedInput( - seed_file=DiscoveredSeedFile( - file_path=Path("/project/seeds/product_types.csv"), - relative_path=Path("seeds/product_types.csv"), - ), - schema_entry=SchemaSeedEntry(name="product_types", description=description), - schema_file=DiscoveredSchemaFile( - file_path=Path("/project/seeds/product_types.yml"), - relative_path=Path("seeds/product_types.yml"), - contents=contents, - model_entries=(), - seed_entries=(), - ), - ), - ) - ) - - -def returns_source_input(*, description: str | None, loader: str | None) -> CompileSourceInput: - return CompileSourceInput( - source_entry=SourceEntry(name="raw_returns", description=description, loader=loader), - source_file=DiscoveredSourceFile( - file_path=Path("/project/sources/raw.yml"), - relative_path=Path("sources/raw.yml"), - contents="sources:\n - name: raw_orders\n description: Orders\n" - " - name: raw_returns\n", - source_entries=(), - ), - ) - - -def source_description_inputs(description: str | None) -> RequiredDescriptionInputs: - return RequiredDescriptionInputs( - source_inputs=(returns_source_input(description=description, loader=None),) - ) - - -def returns_loader(description: str | None) -> DiscoveredLoaderFunction: - return DiscoveredLoaderFunction( - file_path=Path("/project/python/loaders/raw_returns.py"), - relative_path=Path("python/loaders/raw_returns.py"), - name="raw_returns", - function=documented_returns_loader, - description=description, - ) - - -def loader_backed_source_description_inputs(description: str | None) -> RequiredDescriptionInputs: - return RequiredDescriptionInputs( - source_inputs=(returns_source_input(description=None, loader="raw_returns"),), - discovered_inputs=replace( - REQUIRED_DESCRIPTION_BASE_INPUTS, loader_functions=(returns_loader(description),) - ), - ) +def usage_consumers(model_input: ModelInput) -> set[ResourceIdentity | DeclarationIdentity]: + """The consumers a model input's declaration usages are recorded for.""" - -def loader_only_description_inputs(description: str | None) -> RequiredDescriptionInputs: - return RequiredDescriptionInputs( - discovered_inputs=replace( - REQUIRED_DESCRIPTION_BASE_INPUTS, loader_functions=(returns_loader(description),) - ) - ) - - -def function_description_inputs( - description: str | None, - *, - contents: str = "FUNCTION (returns BOOLEAN);\n\nTRUE\n", - relative_path: Path = Path("functions/sql/is_large_order.sql"), - language: FunctionLanguage = FunctionLanguage.SQL, -) -> RequiredDescriptionInputs: - return RequiredDescriptionInputs( - function_inputs=( - CompileSqlFunctionInput( - function_file=DiscoveredSqlFunctionFile( - file_path=Path("/project") / relative_path, - relative_path=relative_path, - contents=contents, - header_values={}, - body_sql="TRUE", - ), - name="is_large_order", - arguments=(), - returns="BOOLEAN", - body_sql="TRUE", - description=description, - language=language, - ), - ) - ) - - -def sql_hook_description_inputs( - description: str | None, *, contents: str = "HOOK ();\nSELECT 1\n" -) -> RequiredDescriptionInputs: - return RequiredDescriptionInputs( - discovered_inputs=replace( - REQUIRED_DESCRIPTION_BASE_INPUTS, - sql_hook_files=( - DiscoveredSqlHookFile( - file_path=Path("/project/hooks/sql/analyze_orders.sql"), - relative_path=Path("hooks/sql/analyze_orders.sql"), - contents=contents, - header_values={}, - sql_body="SELECT 1", - name="analyze_orders", - description=description, - ), - ), - ) - ) - - -def task_description_inputs(description: str | None) -> RequiredDescriptionInputs: - return RequiredDescriptionInputs( - discovered_inputs=replace( - REQUIRED_DESCRIPTION_BASE_INPUTS, - task_functions=( - DiscoveredTaskFunction( - file_path=Path("/project/python/tasks/refresh_exports.py"), - relative_path=Path("python/tasks/refresh_exports.py"), - name="refresh_exports", - function=refresh_exports_task, - description=description, - ), - ), - ) - ) - - -def provider_description_inputs(description: str | None) -> RequiredDescriptionInputs: - provider_class: type[Provider] = _PROVIDER_CLASS_BY_UNDESCRIBED[description is None] - return RequiredDescriptionInputs( - discovered_inputs=replace( - REQUIRED_DESCRIPTION_BASE_INPUTS, - providers=( - DiscoveredProvider( - file_path=Path("/project/providers/orders_api.py"), - relative_path=Path("providers/orders_api.py"), - name="orders_api", - provider_class=provider_class, - settings=provider_class(), - ), - ), - ) - ) - - -def required_description_diagnostics( - inputs: RequiredDescriptionInputs, -) -> tuple[CompilerDiagnostic, ...]: - return missing_description_diagnostics( - discovered_inputs=inputs.discovered_inputs, - model_inputs=inputs.model_inputs, - seed_inputs=inputs.seed_inputs, - source_inputs=inputs.source_inputs, - function_inputs=inputs.function_inputs, - ) - - -_CURSOR_MODEL_PATH: Path = Path("models/finance/daily_revenue.sql") -_CURSOR_MODEL_CONTENTS: str = ( - "MODEL (\n" - ' description "Daily revenue from successful payments",\n' - " materialized incremental,\n" - " incremental_strategy delete_insert,\n" - " cursor revenue_date,\n" - " cursor_type timestamp,\n" - " cursor_grain day,\n" - ");\n\n" - "SELECT CAST('2026-04-01' AS DATE) AS revenue_date, 2850 AS total_revenue_cents\n" -) - - -def cursor_model_input( - *, config: dict[str, object], reference_kinds: tuple[str, ...] -) -> CompileModelInput: - """Build the daily_revenue model input with the given config and reference kinds.""" - - return CompileModelInput( - model_file=DiscoveredSqlModelFile( - file_path=Path("/project") / _CURSOR_MODEL_PATH, - relative_path=_CURSOR_MODEL_PATH, - contents=_CURSOR_MODEL_CONTENTS, - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="", - ), - config=CompileModelConfig(values=dict(config)), - references=tuple( - CompileSqlReference(ref_kind=kind, ref_name=f"input_{index}") - for index, kind in enumerate(reference_kinds) - ), - ) - - -def diagnostic_spans( - diagnostics: tuple[CompilerDiagnostic, ...], -) -> tuple[tuple[str, int, int, int | None], ...]: - """Return each located diagnostic's path, line, column and end column.""" - - locations: tuple[SourceLocation, ...] = tuple( - cast(SourceLocation, diagnostic.location) for diagnostic in diagnostics - ) - return tuple( - (location.path.as_posix(), location.line, location.column, location.end_column) - for location in locations - ) - - -def extract_test_ctes( - *, ctes: tuple[tuple[str, str], ...], file_label: str, mode: SqlTestMode -) -> CompileSqlTestCtes: - """Extract a SQL test made of `ctes` before expansion under generic SQL rules.""" - - return extract_test_sql( - sql="WITH " + ", ".join(f"{name} AS ({body})" for name, body in ctes), - file_label=file_label, - mode=mode, - ) - - -def extract_test_sql( - *, sql: str, file_label: str, mode: SqlTestMode = SqlTestMode.MODEL -) -> CompileSqlTestCtes: - """Extract one SQL test before expansion under generic SQL rules.""" - - return extract_unexpanded_sql_test( - sql=sql, file_label=file_label, mode=mode, syntax=SqlLexicalSyntax() - )[0] + return {usage.consumer for usage in model_input.declaration_usages} def reanalysed_columns( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py index 191fdad6dc..161e693972 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py @@ -27,19 +27,3 @@ class CursorIntrinsicAnalysisTestCase: description: str cursor_type: str expected_type: str - - -@dataclass(frozen=True) -class ParameterizedSqlRenderTestCase: - description: str - sql: str - arguments: dict[str, object] - expected_sql: str - - -@dataclass(frozen=True) -class ParameterizedSqlRenderErrorTestCase: - description: str - sql: str - arguments: dict[str, object] - expected_error_fragment: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_arguments.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_arguments.py deleted file mode 100644 index ac8132cf1a..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_arguments.py +++ /dev/null @@ -1,90 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile._helpers.render.arguments import render_parameterized_sql -from sqlbuild.compiler.compile.exceptions import CompileInputError -from tests.unit.src.sqlbuild.compiler.compile._helpers.render._test_types import ( - ParameterizedSqlRenderErrorTestCase, - ParameterizedSqlRenderTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ParameterizedSqlRenderTestCase( - description="renders raw and quoted values with SQL semantics", - sql="SELECT @relation, @'role', @enabled, @limit, @missing, @'values'", - arguments={ - "relation": "analytics.orders", - "role": "O'Brien", - "enabled": True, - "limit": 3, - "missing": None, - "values": ["reader", "writer"], - }, - expected_sql=("SELECT analytics.orders, 'O''Brien', TRUE, 3, NULL, 'reader', 'writer'"), - ), - ParameterizedSqlRenderTestCase( - description="preserves macro and declaration calls", - sql='SELECT @grant_select(), @enum("role").ADMIN, @const("limit")', - arguments={}, - expected_sql='SELECT @grant_select(), @enum("role").ADMIN, @const("limit")', - ), - ParameterizedSqlRenderTestCase( - description="does not rescan rendered quoted values as raw arguments", - sql="SELECT @'email'", - arguments={"email": "O'Brien@example.com"}, - expected_sql="SELECT 'O''Brien@example.com'", - ), - ], - ids=lambda case: case.description, -) -def test_given_sql_arguments_when_rendering_then_returns_expected_sql( - test_case: ParameterizedSqlRenderTestCase, -) -> None: - rendered: str = render_parameterized_sql( - sql=test_case.sql, - arguments=test_case.arguments, - owner_label="models/orders.sql post_hooks[0]", - definition_label="SQL hook 'grant_access'", - reject_unused=True, - ) - - assert rendered == test_case.expected_sql - - -@pytest.mark.parametrize( - "test_case", - [ - ParameterizedSqlRenderErrorTestCase( - description="rejects missing argument", - sql="SELECT @role", - arguments={}, - expected_error_fragment="missing argument 'role'", - ), - ParameterizedSqlRenderErrorTestCase( - description="rejects unused argument", - sql="SELECT 1", - arguments={"unused": "value"}, - expected_error_fragment=r"unknown argument\(s\).*unused", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_sql_arguments_when_rendering_then_raises( - test_case: ParameterizedSqlRenderErrorTestCase, -) -> None: - with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - render_parameterized_sql( - sql=test_case.sql, - arguments=test_case.arguments, - owner_label="models/orders.sql post_hooks[0]", - definition_label="SQL hook 'grant_access'", - reject_unused=True, - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index e8956a2043..d0eb22c4da 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -10,7 +10,6 @@ from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( cursor_intrinsics_analysis_sql, get_validated_model_cursor_intrinsics, - reject_cursor_intrinsics, render_cursor_intrinsics, ) from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -116,24 +115,6 @@ def test_given_invalid_intrinsic_when_validating_then_raises( ) -@pytest.mark.parametrize( - "test_case", - [ - CursorIntrinsicErrorTestCase( - description="rejects an audit intrinsic", - sql="SELECT __cursor_start()", - expected_error_fragment="only supported", - ) - ], - ids=lambda case: case.description, -) -def test_given_unsupported_sql_context_when_rejecting_then_raises( - test_case: CursorIntrinsicErrorTestCase, -) -> None: - with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - reject_cursor_intrinsics(sql=test_case.sql, context="Audit 'freshness'") - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index f3e0e0a350..41e1f8d240 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -11,13 +11,13 @@ from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, - CompiledSeed, - CompileProjectInputs, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, + CompiledProjectView, CompiledResourceType, + CompiledSeedView, + CompileInputsView, ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry @@ -514,12 +514,12 @@ def test_given_compile_inputs_when_assembling_compiled_project_then_returns_expe monkeypatch.setenv("USE_SEED_DB", "1") write_repo_files(tmp_path, test_case.repo_files) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run_id", ) - compiled: CompiledProject = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) assert tuple(m.name for m in compiled.models) == test_case.expected_model_names assert tuple(m.deps for m in compiled.models) == test_case.expected_model_deps @@ -618,13 +618,13 @@ def test_given_named_generic_audit_instances_when_assembling_then_each_name_is_p + "\n", }, ) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run_id", ) - compiled: CompiledProject = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) assert tuple(audit.name for audit in compiled.audits) == test_case.expected_names @@ -691,12 +691,12 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph }, ) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) - seeds: dict[str, CompiledSeed] = { + seeds: dict[str, CompiledSeedView] = { seed.name: seed for seed in assemble_compiled_project(inputs=compile_inputs).seeds } @@ -715,12 +715,12 @@ def test_given_preserved_target_when_assembling_seed_then_declaration_remains_ph seeds["currencies"].destination.logical_schema, ) == test_case.expected_defaulted_location * 2 - physical_inputs: CompileProjectInputs = build_compile_inputs( + physical_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, selected_target="physical", ) - physical_seeds: tuple[CompiledSeed, ...] = assemble_compiled_project( + physical_seeds: tuple[CompiledSeedView, ...] = assemble_compiled_project( inputs=physical_inputs ).seeds assert {(seed.destination.database, seed.destination.schema) for seed in physical_seeds} == { @@ -793,13 +793,13 @@ def test_given_connection_location_when_assembling_project_then_sets_effective_t base_repo_files() | {"sqlbuild_project.toml": test_case.project_toml}, ) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run_id", ) - compiled: CompiledProject = assemble_compiled_project(inputs=compile_inputs) + compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) assert compiled.effective_target_database == test_case.expected_effective_target_database assert compiled.effective_target_schema == test_case.expected_effective_target_schema @@ -855,13 +855,13 @@ def test_given_sql_hooks_when_assembling_then_applies_adapter_aware_validation( }, ) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run_id", ) - compiled: CompiledProject = assemble_compiled_project( + compiled: CompiledProjectView = assemble_compiled_project( inputs=compile_inputs, inference_profile=ExpressionInferenceProfile( sql_analysis_dialect=test_case.dialect, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py index ded2583fcb..64a06f3b1f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_attached_audit_gates.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AttachedAuditGateCycleTestCase, AttachedAuditGateTestCase, @@ -95,7 +95,7 @@ def test_given_attached_audit_reading_another_resource_when_compiling_then_it_ad ) -> None: write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files) - project: CompiledProject = compile_and_assemble(project_dir=tmp_path) + project: CompiledProjectView = compile_and_assemble(project_dir=tmp_path) assert set(test_case.expected_edges) <= execution_edge_names(project=project) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index 4f1801971c..b6f7ddbcba 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -9,7 +9,7 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( @@ -61,7 +61,7 @@ def unused_quality(): }, ) - inputs: CompileProjectInputs = build_compile_inputs( + inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run", diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_validation.py deleted file mode 100644 index 654fddfb3e..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_validation.py +++ /dev/null @@ -1,285 +0,0 @@ -"""Tests for audit severity resolution, run scope resolution, and attached audit ref validation.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.auditing.types import AuditRunScope -from sqlbuild.compiler.compile._helpers.attachment.audits import ( - resolve_audit_run_scope, - resolve_audit_severity, - validate_model_attached_audit_references, -) -from sqlbuild.compiler.compile._helpers.attachment.references import validate_audit_references -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlReference -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind -from sqlbuild.compiler.discovery.models import DiscoveredAuditFile -from sqlbuild.compiler.references.types import SqlReferenceKind -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - ResolveAuditRunScopeErrorTestCase, - ResolveAuditRunScopeTestCase, - ResolveAuditSeverityTestCase, - ValidateAuditRefsErrorTestCase, - ValidateModelAttachedAuditRefsTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveAuditSeverityTestCase( - description="instance severity wins over project default", - instance_severity="error", - default_severity="warn", - expected_severity="error", - ), - ResolveAuditSeverityTestCase( - description="project default used when instance is None", - instance_severity=None, - default_severity="warn", - expected_severity="warn", - ), - ResolveAuditSeverityTestCase( - description="falls back to error when instance and project default are None", - instance_severity=None, - default_severity=None, - expected_severity="error", - ), - ResolveAuditSeverityTestCase( - description="instance warn is valid", - instance_severity="warn", - default_severity=None, - expected_severity="warn", - ), - ], - ids=lambda case: case.description, -) -def test_given_valid_severity_inputs_when_resolving_then_returns_expected( - test_case: ResolveAuditSeverityTestCase, -) -> None: - result: str = resolve_audit_severity( - instance_severity=test_case.instance_severity, - default_severity=test_case.default_severity, - audit_label="test audit", - ) - - assert result == test_case.expected_severity - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveAuditSeverityTestCase( - description="unknown instance severity raises compile error", - instance_severity="critical", - default_severity=None, - expected_error_fragment="unknown severity", - ), - ResolveAuditSeverityTestCase( - description="unknown default severity raises compile error", - instance_severity=None, - default_severity="critical", - expected_error_fragment="unknown value", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_severity_inputs_when_resolving_then_raises( - test_case: ResolveAuditSeverityTestCase, -) -> None: - assert test_case.expected_error_fragment is not None - - with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - resolve_audit_severity( - instance_severity=test_case.instance_severity, - default_severity=test_case.default_severity, - audit_label="test audit", - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveAuditRunScopeTestCase( - description="instance run_scope wins over project default", - instance_run_scope="delta_and_final", - default_run_scope="final", - expected_run_scope="delta_and_final", - ), - ResolveAuditRunScopeTestCase( - description="project default used when instance is None", - instance_run_scope=None, - default_run_scope="delta_and_final", - expected_run_scope="delta_and_final", - ), - ResolveAuditRunScopeTestCase( - description="falls back to delta_and_final when both are None", - instance_run_scope=None, - default_run_scope=None, - expected_run_scope=AuditRunScope.DELTA_AND_FINAL, - ), - ], - ids=lambda case: case.description, -) -def test_given_valid_run_scope_inputs_when_resolving_then_returns_expected( - test_case: ResolveAuditRunScopeTestCase, -) -> None: - result: str = resolve_audit_run_scope( - instance_run_scope=test_case.instance_run_scope, - default_run_scope=test_case.default_run_scope, - ) - - assert result == test_case.expected_run_scope - - -@pytest.mark.parametrize( - "test_case", - [ - ResolveAuditRunScopeErrorTestCase( - description="unknown instance run_scope raises compile error", - instance_run_scope="delta_only", - default_run_scope=None, - expected_error_fragment="unknown audit run_scope", - ), - ResolveAuditRunScopeErrorTestCase( - description="unknown default run_scope raises compile error", - instance_run_scope=None, - default_run_scope="delta_only", - expected_error_fragment="unknown value", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_run_scope_inputs_when_resolving_then_raises( - test_case: ResolveAuditRunScopeErrorTestCase, -) -> None: - with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - resolve_audit_run_scope( - instance_run_scope=test_case.instance_run_scope, - default_run_scope=test_case.default_run_scope, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ValidateModelAttachedAuditRefsTestCase( - description="model-attached audit referencing attached model passes", - references=(CompileSqlReference(ref_kind=SqlReferenceKind.REF, ref_name="orders"),), - attached_target_kind=AttachedAuditTargetKind.MODEL, - attached_target_name="orders", - ), - ValidateModelAttachedAuditRefsTestCase( - description="source-attached audit skips model ref check", - references=( - CompileSqlReference(ref_kind=SqlReferenceKind.SOURCE, ref_name="raw_orders"), - ), - attached_target_kind=AttachedAuditTargetKind.SOURCE, - attached_target_name="raw_orders", - ), - ValidateModelAttachedAuditRefsTestCase( - description="model-attached audit with multiple refs including attached model passes", - references=( - CompileSqlReference(ref_kind=SqlReferenceKind.REF, ref_name="orders"), - CompileSqlReference(ref_kind=SqlReferenceKind.SOURCE, ref_name="raw_data"), - ), - attached_target_kind=AttachedAuditTargetKind.MODEL, - attached_target_name="orders", - ), - ], - ids=lambda case: case.description, -) -def test_given_valid_attached_audit_refs_when_validating_then_passes( - test_case: ValidateModelAttachedAuditRefsTestCase, -) -> None: - validate_model_attached_audit_references( - references=test_case.references, - attached_target_kind=test_case.attached_target_kind, - attached_target_name=test_case.attached_target_name, - audit_label="test audit", - ) - - assert test_case.expected_valid - - -@pytest.mark.parametrize( - "test_case", - [ - ValidateModelAttachedAuditRefsTestCase( - description="model-attached audit without ref to attached model raises", - references=( - CompileSqlReference(ref_kind=SqlReferenceKind.SOURCE, ref_name="raw_orders"), - ), - attached_target_kind=AttachedAuditTargetKind.MODEL, - attached_target_name="orders", - expected_valid=False, - expected_error_fragment="must reference the attached model", - ), - ValidateModelAttachedAuditRefsTestCase( - description="model-attached audit referencing wrong model raises", - references=(CompileSqlReference(ref_kind=SqlReferenceKind.REF, ref_name="customers"),), - attached_target_kind=AttachedAuditTargetKind.MODEL, - attached_target_name="orders", - expected_valid=False, - expected_error_fragment="must reference the attached model", - ), - ValidateModelAttachedAuditRefsTestCase( - description="model-attached audit with no refs at all raises", - references=(), - attached_target_kind=AttachedAuditTargetKind.MODEL, - attached_target_name="orders", - expected_valid=False, - expected_error_fragment="must reference the attached model", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_attached_audit_refs_when_validating_then_raises( - test_case: ValidateModelAttachedAuditRefsTestCase, -) -> None: - assert test_case.expected_error_fragment is not None - - with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - validate_model_attached_audit_references( - references=test_case.references, - attached_target_kind=test_case.attached_target_kind, - attached_target_name=test_case.attached_target_name, - audit_label="test audit", - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ValidateAuditRefsErrorTestCase( - description="dbt ref in audit raises compile error", - references=(CompileSqlReference(ref_kind=SqlReferenceKind.DBT_REF, ref_name="orders"),), - expected_error_fragment="audit dbt model checks belong in dbt", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_audit_refs_when_validating_then_raises( - test_case: ValidateAuditRefsErrorTestCase, -) -> None: - audit_file: DiscoveredAuditFile = DiscoveredAuditFile( - file_path=Path("audits/dbt_model_check.sql"), - relative_path=Path("audits/dbt_model_check.sql"), - contents="", - blocks=(), - ) - - with pytest.raises( - CompileInputError, - match=test_case.expected_error_fragment, - ): - validate_audit_references( - references=test_case.references, - audit_file=audit_file, - known_model_names=set(), - known_seed_names=set(), - known_source_names=set(), - ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_model_inputs.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_model_inputs.py deleted file mode 100644 index 02041d5323..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_cursor_model_inputs.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Unit tests for the compile check that rejects cursor models without inputs (P011).""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile._helpers.config.cursor_inputs import ( - cursor_model_without_inputs_diagnostics, -) -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - CursorModelWithoutInputsCase, -) -from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( - cursor_model_input, - diagnostic_spans, -) - -_CURSOR_CONFIG: dict[str, object] = { - "materialized": "incremental", - "incremental_strategy": "delete_insert", - "cursor": "revenue_date", - "cursor_type": "timestamp", - "cursor_grain": "day", -} -_CURSOR_LOCATION: tuple[str, int, int, int] = ("models/finance/daily_revenue.sql", 5, 3, 22) -_MESSAGE: str = ( - "incremental model 'daily_revenue' reads no inputs, so builds after the first " - "cannot work out their cursor window" -) - - -@pytest.mark.parametrize( - "test_case", - [ - CursorModelWithoutInputsCase( - description="cursor model reading only constants", - config=_CURSOR_CONFIG, - reference_kinds=(), - expected_codes=("P011",), - expected_locations=(_CURSOR_LOCATION,), - ), - CursorModelWithoutInputsCase( - description="cursor model calling only a function", - config=_CURSOR_CONFIG, - reference_kinds=("udf",), - expected_codes=("P011",), - expected_locations=(_CURSOR_LOCATION,), - ), - CursorModelWithoutInputsCase( - description="cursor model reading a ref", - config=_CURSOR_CONFIG, - reference_kinds=("ref",), - expected_codes=(), - ), - CursorModelWithoutInputsCase( - description="cursor model reading a source", - config=_CURSOR_CONFIG, - reference_kinds=("source",), - expected_codes=(), - ), - CursorModelWithoutInputsCase( - description="cursor model reading a seed", - config=_CURSOR_CONFIG, - reference_kinds=("seed",), - expected_codes=(), - ), - CursorModelWithoutInputsCase( - description="cursor model with declared cursor inputs", - config={**_CURSOR_CONFIG, "cursor_inputs": {"payments": "paid_at"}}, - reference_kinds=(), - expected_codes=(), - ), - CursorModelWithoutInputsCase( - description="rolling window microbatch model", - config={ - **_CURSOR_CONFIG, - "incremental_mode": "microbatch", - "microbatch_strategy": "rolling_window", - }, - reference_kinds=(), - expected_codes=(), - ), - CursorModelWithoutInputsCase( - description="incremental model without a cursor", - config={"materialized": "incremental", "incremental_strategy": "append"}, - reference_kinds=(), - expected_codes=(), - ), - CursorModelWithoutInputsCase( - description="table model", - config={"materialized": "table"}, - reference_kinds=(), - expected_codes=(), - ), - ], - ids=lambda case: case.description, -) -def test_given_model_inputs_when_checking_cursor_inputs_then_reports_p011_only_without_inputs( - test_case: CursorModelWithoutInputsCase, -) -> None: - diagnostics: tuple[CompilerDiagnostic, ...] = cursor_model_without_inputs_diagnostics( - model_inputs=( - cursor_model_input(config=test_case.config, reference_kinds=test_case.reference_kinds), - ) - ) - - assert tuple(diagnostic.code for diagnostic in diagnostics) == test_case.expected_codes - assert all(diagnostic.message == _MESSAGE for diagnostic in diagnostics) - assert all( - "__ref(), __source() or __seed()" in (diagnostic.help or "") for diagnostic in diagnostics - ) - assert diagnostic_spans(diagnostics) == test_case.expected_locations diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py index e2fe64e774..40097e240e 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declaration_scopes.py @@ -9,15 +9,10 @@ import pytest -from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import ( - CompiledProject, - CompileModelInput, - CompileProjectInputs, -) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView, ModelInput from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, @@ -29,13 +24,11 @@ DeclarationIdentity, ResourceIdentity, UsageRecord, - VisibilityRecord, ) from sqlbuild.compiler.scopes.types import ( DeclarationKind, ResourceKind, UsageKind, - VisibilityReason, ) from sqlbuild.spec.contracts.models import SourceEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( @@ -51,7 +44,7 @@ DUCKDB_COMPILE_ADAPTER_CONTEXT, compile_project_inputs, render_compile_diagnostics, - visible_declarations_without_runtime_values, + usage_consumers, ) _PROJECT_FILE: str = """ @@ -64,67 +57,49 @@ """ -@pytest.mark.parametrize( - "test_case", - [ExpectedBooleanTestCase(description="immutable facts are reused", expected_result=True)], - ids=lambda case: case.description, -) -def test_given_no_visibility_records_when_rebinding_then_reuses_immutable_facts( - test_case: ExpectedBooleanTestCase, -) -> None: - declarations: attachment_core._VisibleModelDeclarations = ( - visible_declarations_without_runtime_values() - ) - - rebound: attachment_core._VisibleModelDeclarations = ( - attachment_core._rebind_visible_declarations( - declarations=declarations, - consumer=ResourceIdentity(ResourceKind.MODEL, "orders"), - ) - ) - - assert (rebound is declarations) is test_case.expected_result - - @pytest.mark.parametrize( "test_case", [ ExpectedBooleanTestCase( - description="new consumer gets independent facts", expected_result=True + description="models sharing a folder each record their own enum usage", + expected_result=True, ) ], ids=lambda case: case.description, ) -def test_given_visibility_record_when_rebinding_then_projects_new_consumer_independently( +def test_given_models_in_one_folder_when_compiling_then_folder_visibility_is_rebound_per_model( test_case: ExpectedBooleanTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: - original_consumer: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, "customers") - new_consumer: ResourceIdentity = ResourceIdentity(ResourceKind.MODEL, "orders") - declaration: DeclarationIdentity = DeclarationIdentity(DeclarationKind.ENUM, "status") - declarations: attachment_core._VisibleModelDeclarations = ( - visible_declarations_without_runtime_values( - enum_visibility={ - "status": ( - VisibilityRecord( - resource=original_consumer, - declaration=declaration, - reason=VisibilityReason.GLOBAL, - ), - ) - } - ) + write_repo_files( + tmp_path, + { + "sqlbuild_project.toml": _PROJECT_FILE, + "models/domain/_enums/status.sql": "ENUM (name status, members [OPEN, SHIPPED]);", + "models/domain/customers.sql": ( + 'MODEL (description "Test model customers.");\n' + 'SELECT @enum("status").OPEN AS status\n' + ), + "models/domain/orders.sql": ( + 'MODEL (description "Test model orders.");\n' + 'SELECT @enum("status").SHIPPED AS status\n' + ), + }, ) - rebound: attachment_core._VisibleModelDeclarations = ( - attachment_core._rebind_visible_declarations( - declarations=declarations, - consumer=new_consumer, - ) - ) + inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) - assert (rebound is not declarations) is test_case.expected_result - assert rebound.enum_visibility["status"][0].resource == new_consumer - assert declarations.enum_visibility["status"][0].resource == original_consumer + consumers: list[set[ResourceIdentity | DeclarationIdentity]] = [ + usage_consumers(model_input) for model_input in inputs.model_inputs + ] + assert ( + consumers + == [ + {ResourceIdentity(ResourceKind.MODEL, "customers")}, + {ResourceIdentity(ResourceKind.MODEL, "orders")}, + ] + ) is test_case.expected_result @pytest.mark.parametrize( @@ -187,7 +162,7 @@ def test_given_scoped_declaration_when_compiling_model_then_only_lexically_visib | {test_case.model_path: test_case.model_sql}, ) - inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) + inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) assert inputs.model_inputs[0].query_sql == test_case.expected_sql assert inputs.scope_index.declarations @@ -384,7 +359,7 @@ def test_given_scoped_declaration_when_compiling_sql_surface_then_uses_authored_ ) -> None: write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files) - inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) + inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) actual: str | None = test_case.result(inputs) assert actual == test_case.expected_sql @@ -548,9 +523,9 @@ def test_given_expected_models_when_compiling_then_public_declarations_are_grant ) -> None: write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files) - first: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) - second: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) - compiled: CompiledProject = assemble_project(inputs=first, skip_column_inference=True) + first: CompileInputsView = compile_project_inputs(project_dir=tmp_path) + second: CompileInputsView = compile_project_inputs(project_dir=tmp_path) + compiled: CompiledProjectView = assemble_project(inputs=first, skip_column_inference=True) sql_body: str = getattr(first, test_case.input_collection)[0].sql_body grants: tuple[tuple[str, str, str], ...] = tuple( sorted( @@ -607,8 +582,8 @@ def test_given_relationship_granted_reference_when_compiling_then_usage_retains_ {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files, ) - inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) - compiled: CompiledProject = assemble_project(inputs=inputs, skip_column_inference=True) + inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) + compiled: CompiledProjectView = assemble_project(inputs=inputs, skip_column_inference=True) assert test_case.expected_usage in compiled.scope_index.usages @@ -644,8 +619,8 @@ def test_given_exact_declaration_placement_when_assembling_then_project_is_accep {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files, ) - inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) - compiled: CompiledProject = assemble_project(inputs=inputs, skip_column_inference=True) + inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) + compiled: CompiledProjectView = assemble_project(inputs=inputs, skip_column_inference=True) assert compiled.scope_index.completeness.runtime_usage is test_case.expected_complete assert compiled.scope_index.completeness.placement is test_case.expected_complete @@ -677,8 +652,8 @@ def test_given_models_in_same_directory_when_reusing_visibility_then_usage_consu }, ) - inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) - compiled: CompiledProject = assemble_project(inputs=inputs, skip_column_inference=True) + inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) + compiled: CompiledProjectView = assemble_project(inputs=inputs, skip_column_inference=True) consumers: set[ResourceIdentity] = { cast(ResourceIdentity, usage.consumer) for usage in compiled.scope_index.usages @@ -734,9 +709,9 @@ def test_given_invalid_declaration_placement_when_assembling_then_it_reports_an_ tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files, ) - inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) + inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) - project: CompiledProject = assemble_project(inputs=inputs, skip_column_inference=True) + project: CompiledProjectView = assemble_project(inputs=inputs, skip_column_inference=True) assert re.search( cast(str, test_case.expected_fragment), @@ -769,9 +744,9 @@ def test_given_disabled_placement_enforcement_when_assembling_then_project_and_w tmp_path, {"sqlbuild_project.toml": project_file} | test_case.files, ) - inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) + inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) - compiled: CompiledProject = assemble_project(inputs=inputs, skip_column_inference=True) + compiled: CompiledProjectView = assemble_project(inputs=inputs, skip_column_inference=True) assert tuple(model.name for model in compiled.models) == test_case.expected_model_names assert ( @@ -813,7 +788,7 @@ def test_given_inherited_enum_when_compiling_contract_then_resolves_from_model_p | {test_case.model_path: test_case.model_sql}, ) - model: CompileModelInput = compile_project_inputs(project_dir=tmp_path).model_inputs[0] + model: ModelInput = compile_project_inputs(project_dir=tmp_path).model_inputs[0] assert model.query_sql == test_case.expected_sql assert model.schema_entry is not None @@ -857,7 +832,7 @@ def test_given_scoped_declaration_when_compiling_source_then_uses_authored_sourc ), ) - inputs: CompileProjectInputs = build_compile_inputs( + inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="source_scope_test", @@ -907,7 +882,7 @@ def test_given_sources_in_same_file_when_reusing_visibility_then_usage_consumers ), ) - inputs: CompileProjectInputs = build_compile_inputs( + inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="source_scope_reuse_test", @@ -953,7 +928,7 @@ def test_given_tests_in_same_directory_when_reusing_visibility_then_usage_consum }, ) - inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) + inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) test_usages: Iterator[UsageRecord] = chain.from_iterable( test.declaration_usages for test in inputs.test_inputs diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py index 01b5e45917..ce03a1beb0 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_declarations.py @@ -10,11 +10,11 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, - CompileModelInput, - CompileProjectInputs, +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompileInputsView, + ModelInput, ) from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.main.discover import discover_project_inputs @@ -152,13 +152,13 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop }, ) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run", ) - model_input: CompileModelInput = compile_inputs.model_inputs[0] + model_input: ModelInput = compile_inputs.model_inputs[0] assert model_input.schema_entry is not None assert model_input.query_sql == test_case.expected_query_sql assert tuple(compile_inputs.public_enums) == ("fulfillment_method", "priority") @@ -210,11 +210,11 @@ def test_given_public_and_private_declarations_when_compiling_then_resolves_scop "WITH __ref__orders AS (\n SELECT 7 AS threshold\n),\n" "__expected__orders AS (\n SELECT 7 AS threshold\n)\nSELECT 1", ) - compiled_project: CompiledProject = assemble_project( + compiled_project: CompiledProjectView = assemble_project( inputs=compile_inputs, skip_column_inference=True, ) - compiled_model: CompiledModel = compiled_project.models[0] + compiled_model: CompiledModelView = compiled_project.models[0] assert tuple(compiled_project.public_enums) == ("fulfillment_method", "priority") assert tuple(compiled_project.public_constants) == ("min_items", "source_name") assert tuple(compiled_model.enum_columns) == ("fulfillment_method", "priority", "state") @@ -377,7 +377,7 @@ def test_given_typed_constants_when_compiling_then_all_sql_surfaces_use_adapter_ }, ) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter_context=DUCKDB_ARRAY_COMPILE_ADAPTER_CONTEXT, run_id="typed_constants", @@ -526,13 +526,13 @@ def test_given_declaration_change_when_compiling_then_updates_dependent_identity } | test_case.additional_files, ) - initial_model: CompiledModel = compile_first_model(project_dir=tmp_path) + initial_model: CompiledModelView = compile_first_model(project_dir=tmp_path) write_repo_files( tmp_path, {test_case.declaration_path: test_case.changed_declaration}, ) - changed_model: CompiledModel = compile_first_model(project_dir=tmp_path) + changed_model: CompiledModelView = compile_first_model(project_dir=tmp_path) query_hash_changed: bool = compute_query_hash( query_sql=initial_model.query_sql, dialect=None diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py index 79ecef24b8..ac13c897b2 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_descriptions.py @@ -1,11 +1,8 @@ -"""Unit tests for the required-description compile check (P010).""" +"""Every discovered input kind is either checked by P010 or explicitly exempt.""" from __future__ import annotations -import inspect -from dataclasses import asdict, fields -from functools import partial -from pathlib import Path +from dataclasses import fields import pytest @@ -13,138 +10,12 @@ DESCRIPTION_EXEMPT_INPUTS, DESCRIPTION_REQUIRED_INPUT_KINDS, ) -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( DescriptionInputClassificationCase, - RequiredDescriptionLocationCase, - RequiredDescriptionUnitCase, -) -from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( - UndescribedApi, - documented_returns_loader, - function_description_inputs, - loader_backed_source_description_inputs, - loader_only_description_inputs, - model_description_inputs, - provider_description_inputs, - refresh_exports_task, - required_description_diagnostics, - scenario_description_inputs, - seed_description_inputs, - source_description_inputs, - sql_hook_description_inputs, - task_description_inputs, -) - -_UDF_AFTER_MENTIONS: str = ( - '"""Mentions @udf before the decorator."""\n' - "# @udf in a comment\n" - "from sqlbuild.functions import udf\n\n\n" - '@udf(returns="BOOLEAN")\n' - "def main(amount: int) -> bool:\n" - " return amount > 100\n" -) - - -CASES: tuple[RequiredDescriptionUnitCase, ...] = ( - RequiredDescriptionUnitCase( - "model header after a leading comment", - model_description_inputs, - "model 'order_totals' has no description", - ("models/order_totals.sql", 2, 1), - ), - RequiredDescriptionUnitCase( - "scenario header", - scenario_description_inputs, - "scenario 'orders__paid' has no description", - ("tests/scenarios/orders__paid.sql", 2, 1), - ), - RequiredDescriptionUnitCase( - "seed YAML entry", - seed_description_inputs, - "seed 'product_types' has no description", - ("seeds/product_types.yml", 2, 1), - ), - RequiredDescriptionUnitCase( - "source YAML entry among others", - source_description_inputs, - "source 'raw_returns' has no description", - ("sources/raw.yml", 4, 1), - ), - RequiredDescriptionUnitCase( - "source described through its loader", - loader_backed_source_description_inputs, - "source 'raw_returns' has no description", - ("sources/raw.yml", 4, 1), - ), - RequiredDescriptionUnitCase( - "loader without a source declaration", - loader_only_description_inputs, - "loader 'raw_returns' has no description", - ("python/loaders/raw_returns.py", documented_returns_loader.__code__.co_firstlineno, 1), - ), - RequiredDescriptionUnitCase( - "SQL function header", - function_description_inputs, - "function 'is_large_order' has no description", - ("functions/sql/is_large_order.sql", 1, 1), - ), - RequiredDescriptionUnitCase( - "named SQL hook", - sql_hook_description_inputs, - "hook 'analyze_orders' has no description", - ("hooks/sql/analyze_orders.sql", 1, 1), - ), - RequiredDescriptionUnitCase( - "task", - task_description_inputs, - "task 'refresh_exports' has no description", - ("python/tasks/refresh_exports.py", refresh_exports_task.__code__.co_firstlineno, 1), - ), - RequiredDescriptionUnitCase( - "provider", - provider_description_inputs, - "provider 'orders_api' has no description", - ("providers/orders_api.py", inspect.getsourcelines(UndescribedApi)[1], 1), - ), ) -@pytest.mark.parametrize( - "test_case", - [RequiredDescriptionUnitCase(**asdict(case)) for case in CASES], - ids=lambda case: case.description, -) -def test_given_undescribed_resource_when_checking_then_reports_one_p010_at_declaration( - test_case: RequiredDescriptionUnitCase, -) -> None: - diagnostics: tuple[CompilerDiagnostic, ...] = required_description_diagnostics( - test_case.build(None) - ) - - assert [ - (item.code, item.message, (str(item.path), item.line, item.column)) for item in diagnostics - ] == [("P010", test_case.expected_message, test_case.expected_location)] - - -@pytest.mark.parametrize( - "test_case", - [RequiredDescriptionUnitCase(**asdict(case)) for case in CASES], - ids=lambda case: case.description, -) -def test_given_described_resource_when_checking_then_reports_nothing( - test_case: RequiredDescriptionUnitCase, -) -> None: - diagnostics: tuple[CompilerDiagnostic, ...] = required_description_diagnostics( - test_case.build("Described resource.") - ) - - assert test_case.expected_message not in [item.message for item in diagnostics] - assert diagnostics == () - - @pytest.mark.parametrize( "test_case", [ @@ -164,66 +35,5 @@ def test_given_discovered_input_fields_when_classifying_then_every_field_is_chec assert test_case.required | test_case.exempt == test_case.expected_fields -@pytest.mark.parametrize( - "test_case", - [ - RequiredDescriptionLocationCase( - description="MODEL header after a block comment that mentions MODEL (", - build=partial( - model_description_inputs, - None, - contents="/*\nMODEL (old)\n*/\n-- note\nMODEL (materialized table);\nSELECT 1\n", - ), - expected_location=("models/order_totals.sql", 5, 1), - ), - RequiredDescriptionLocationCase( - description="FUNCTION header after a leading comment", - build=partial( - function_description_inputs, - None, - contents="-- returns a flag\nFUNCTION (returns BOOLEAN);\n\nTRUE\n", - ), - expected_location=("functions/sql/is_large_order.sql", 2, 1), - ), - RequiredDescriptionLocationCase( - description="HOOK header after a leading block comment", - build=partial( - sql_hook_description_inputs, None, contents="/* analyze */\nHOOK ();\nSELECT 1\n" - ), - expected_location=("hooks/sql/analyze_orders.sql", 2, 1), - ), - RequiredDescriptionLocationCase( - description="SCENARIO header after a leading comment", - build=partial( - scenario_description_inputs, None, contents="-- paid\nSCENARIO ();\nSELECT 1\n" - ), - expected_location=("tests/scenarios/orders__paid.sql", 2, 1), - ), - RequiredDescriptionLocationCase( - description="@udf decorator after docstring and comment mentions", - build=partial( - function_description_inputs, - None, - contents=_UDF_AFTER_MENTIONS, - relative_path=Path("functions/python/is_large_order.py"), - language=FunctionLanguage.PYTHON, - ), - expected_location=("functions/python/is_large_order.py", 6, 1), - ), - ], - ids=lambda case: case.description, -) -def test_given_comments_before_declaration_when_checking_then_points_at_declaration( - test_case: RequiredDescriptionLocationCase, -) -> None: - diagnostics: tuple[CompilerDiagnostic, ...] = required_description_diagnostics( - test_case.build() - ) - - assert [(str(item.path), item.line, item.column) for item in diagnostics] == [ - test_case.expected_location - ] - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py deleted file mode 100644 index b99112888d..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py +++ /dev/null @@ -1,273 +0,0 @@ -"""Characterisation tests for macro, UDF, and table_fn SQL-test CTE classification errors. - -Duplicate actual or expected CTEs are duplicate CTE names, which the CTE scan rejects first. -""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - ClassifyDirectLogicSqlTestCtesErrorTestCase, -) -from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import extract_test_ctes - -_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() - -_FILE: str = "tests/unit/orders.sql" - - -@pytest.mark.parametrize( - "test_case", - [ - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro duplicate actual", - mode=SqlTestMode.MACRO, - ctes=(("__macro_actual__", "SELECT 1"), ("__macro_actual__", "SELECT 2")), - expected_message=f"SQL test '{_FILE}' defines duplicate CTE '__macro_actual__'", - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro duplicate expected", - mode=SqlTestMode.MACRO, - ctes=( - ("__macro_expected__", "SELECT 1 AS value"), - ("__macro_expected__", "SELECT 2 AS value"), - ), - expected_message=f"SQL test '{_FILE}' defines duplicate CTE '__macro_expected__'", - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro model-test cte", - mode=SqlTestMode.MACRO, - ctes=(("__ref__orders", "SELECT 1"),), - expected_message=( - f"SQL test '{_FILE}' is mode 'macro' but defines model-test CTE '__ref__orders'" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro udf-test cte", - mode=SqlTestMode.MACRO, - ctes=(("__udf_actual__", "SELECT 1"),), - expected_message=( - f"SQL test '{_FILE}' is mode 'macro' but defines UDF-test CTE '__udf_actual__'" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro table_fn-test cte", - mode=SqlTestMode.MACRO, - ctes=(("__table_fn_expected__", "SELECT 1 AS value"),), - expected_message=( - f"SQL test '{_FILE}' is mode 'macro' but defines table_fn-test CTE " - "'__table_fn_expected__'" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro reserved helper name", - mode=SqlTestMode.MACRO, - ctes=(("__missing__", "SELECT 1"),), - expected_message=f"SQL test '{_FILE}' uses reserved helper CTE name '__missing__'", - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro missing expected", - mode=SqlTestMode.MACRO, - ctes=(("__macro_actual__", "SELECT 1"),), - expected_message=( - f"SQL test '{_FILE}' mode 'macro' must define exactly one __macro_actual__ CTE " - "and exactly one __macro_expected__ CTE" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro helper calls macro", - mode=SqlTestMode.MACRO, - ctes=( - ("helper", "SELECT @normalize_status('paid') AS status"), - ("__macro_actual__", "SELECT 1 AS status"), - ("__macro_expected__", "SELECT 1 AS status"), - ), - expected_message=( - f"SQL test '{_FILE}' mode 'macro' helper CTE 'helper' must not call macros; " - "call macros only in __macro_actual__" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro expected calls macro", - mode=SqlTestMode.MACRO, - ctes=( - ("__macro_actual__", "SELECT 1 AS status"), - ("__macro_expected__", "SELECT @normalize_status('paid') AS status"), - ), - expected_message=( - f"SQL test '{_FILE}' mode 'macro' CTE __macro_expected__ must not call macros" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="macro expected unaliased projection names the expected cte", - mode=SqlTestMode.MACRO, - ctes=( - ("__macro_actual__", "SELECT 1 AS status"), - ("__macro_expected__", "SELECT 1 + 1"), - ), - expected_message=( - f"SQL test '{_FILE}' must alias every non-trivial __macro_expected__ projection" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="table_fn expected branch mismatch names the expected cte", - mode=SqlTestMode.TABLE_FN, - ctes=( - ("__table_fn_actual__", "SELECT 1 AS value"), - ("__table_fn_expected__", "SELECT 1 AS value UNION ALL SELECT 2 AS other"), - ), - expected_message=( - f"SQL test '{_FILE}' must use the same __table_fn_expected__ projection names " - "and order in every set-operation branch; branch 2 does not match branch 1" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="udf duplicate actual", - mode=SqlTestMode.UDF, - ctes=(("__udf_actual__", "SELECT 1"), ("__udf_actual__", "SELECT 2")), - expected_message=f"SQL test '{_FILE}' defines duplicate CTE '__udf_actual__'", - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="udf duplicate expected", - mode=SqlTestMode.UDF, - ctes=( - ("__udf_expected__", "SELECT 1 AS value"), - ("__udf_expected__", "SELECT 2 AS value"), - ), - expected_message=f"SQL test '{_FILE}' defines duplicate CTE '__udf_expected__'", - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="udf model-test cte", - mode=SqlTestMode.UDF, - ctes=(("__expected__orders", "SELECT 1"),), - expected_message=( - f"SQL test '{_FILE}' is mode 'udf' but defines model-test CTE '__expected__orders'" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="udf macro-test cte", - mode=SqlTestMode.UDF, - ctes=(("__macro_expected__", "SELECT 1 AS value"),), - expected_message=( - f"SQL test '{_FILE}' is mode 'udf' but defines macro-test CTE '__macro_expected__'" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="udf table_fn-test cte", - mode=SqlTestMode.UDF, - ctes=(("__table_fn_actual__", "SELECT 1"),), - expected_message=( - f"SQL test '{_FILE}' is mode 'udf' but defines table_fn-test CTE " - "'__table_fn_actual__'" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="udf reserved helper name", - mode=SqlTestMode.UDF, - ctes=(("__actual", "SELECT 1"),), - expected_message=f"SQL test '{_FILE}' uses reserved helper CTE name '__actual'", - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="udf missing actual", - mode=SqlTestMode.UDF, - ctes=(("__udf_expected__", "SELECT 1 AS value"),), - expected_message=( - f"SQL test '{_FILE}' mode 'udf' must define exactly one __udf_actual__ CTE " - "and exactly one __udf_expected__ CTE" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="udf helper calls macro", - mode=SqlTestMode.UDF, - ctes=( - ("helper", "SELECT @normalize_status('paid') AS status"), - ("__udf_actual__", "SELECT 1 AS formatted"), - ("__udf_expected__", "SELECT 1 AS formatted"), - ), - expected_message=( - f"SQL test '{_FILE}' mode 'udf' helper CTE 'helper' must not call macros" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="udf expected calls udf", - mode=SqlTestMode.UDF, - ctes=( - ("__udf_actual__", "SELECT 1 AS formatted"), - ("__udf_expected__", 'SELECT __udf("format_cents")(1250) AS formatted'), - ), - expected_message=( - f"SQL test '{_FILE}' mode 'udf' CTE __udf_expected__ must not call udf; " - "call reusable logic only in __udf_actual__" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="table_fn duplicate actual", - mode=SqlTestMode.TABLE_FN, - ctes=(("__table_fn_actual__", "SELECT 1"), ("__table_fn_actual__", "SELECT 2")), - expected_message=f"SQL test '{_FILE}' defines duplicate CTE '__table_fn_actual__'", - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="table_fn duplicate expected", - mode=SqlTestMode.TABLE_FN, - ctes=( - ("__table_fn_expected__", "SELECT 1 AS value"), - ("__table_fn_expected__", "SELECT 2 AS value"), - ), - expected_message=f"SQL test '{_FILE}' defines duplicate CTE '__table_fn_expected__'", - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="table_fn model-test cte", - mode=SqlTestMode.TABLE_FN, - ctes=(("__source__raw_orders", "SELECT 1"),), - expected_message=( - f"SQL test '{_FILE}' is mode 'table_fn' but defines model-test CTE " - "'__source__raw_orders'" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="table_fn macro cte uses generic direct-logic wording", - mode=SqlTestMode.TABLE_FN, - ctes=(("__macro_actual__", "SELECT 1"),), - expected_message=( - f"SQL test '{_FILE}' is mode 'table_fn' but defines another direct-logic CTE " - "'__macro_actual__'" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="table_fn udf cte uses generic direct-logic wording", - mode=SqlTestMode.TABLE_FN, - ctes=(("__udf_expected__", "SELECT 1 AS value"),), - expected_message=( - f"SQL test '{_FILE}' is mode 'table_fn' but defines another direct-logic CTE " - "'__udf_expected__'" - ), - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="table_fn reserved helper name", - mode=SqlTestMode.TABLE_FN, - ctes=(("__unexpected__", "SELECT 1"),), - expected_message=f"SQL test '{_FILE}' uses reserved helper CTE name '__unexpected__'", - ), - ClassifyDirectLogicSqlTestCtesErrorTestCase( - description="table_fn missing both", - mode=SqlTestMode.TABLE_FN, - ctes=(("helper", "SELECT 1"),), - expected_message=( - f"SQL test '{_FILE}' mode 'table_fn' must define exactly one " - "__table_fn_actual__ CTE and exactly one __table_fn_expected__ CTE" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_direct_logic_ctes_when_classifying_then_raises_mode_specific_message( - test_case: ClassifyDirectLogicSqlTestCtesErrorTestCase, -) -> None: - with pytest.raises(CompileInputError) as error_info: - _ = extract_test_ctes(ctes=test_case.ctes, file_label=_FILE, mode=test_case.mode) - - assert str(error_info.value) == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py index 9e8b4fccbe..5b6c6fde24 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_hook_reads.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( HookReadEdgeTestCase, HookReadErrorTestCase, @@ -123,7 +123,7 @@ def test_given_hook_reads_when_compiling_then_hooked_model_runs_after_reads( ) -> None: write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files) - project: CompiledProject = compile_and_assemble(project_dir=tmp_path) + project: CompiledProjectView = compile_and_assemble(project_dir=tmp_path) assert set(test_case.expected_edges) <= execution_edge_names(project=project) @@ -152,7 +152,7 @@ def test_given_sql_hook_read_when_compiling_then_lineage_and_selection_ignore_it ) -> None: write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files) - project: CompiledProject = compile_and_assemble(project_dir=tmp_path) + project: CompiledProjectView = compile_and_assemble(project_dir=tmp_path) assert set(test_case.expected_edges) <= execution_edge_names(project=project) assert set(test_case.expected_edges).isdisjoint(lineage_edge_names(project=project)) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py index 71a1e83e07..e4eec95179 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_schemas.py @@ -12,11 +12,13 @@ from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, - CompileProjectInputs, CompilerDiagnostic, ) +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompileInputsView, +) from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -82,7 +84,7 @@ def test_given_identical_contract_headers_when_attaching_then_each_model_keeps_i }, ) - inputs: CompileProjectInputs = build_compile_inputs( + inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) @@ -223,15 +225,15 @@ def test_given_inherited_model_schemas_when_compiling_then_attaches_resolved_con tuple(str(schema_file.relative_path) for schema_file in discovered.model_schema_files) == test_case.expected_schema_paths ) - inputs: CompileProjectInputs = build_compile_inputs( + inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, run_id="test_run", ) - project: CompiledProject = assemble_project(inputs=inputs) - models: dict[str, CompiledModel] = {model.name: model for model in project.models} + project: CompiledProjectView = assemble_project(inputs=inputs) + models: dict[str, CompiledModelView] = {model.name: model for model in project.models} - exact: CompiledModel = models["exact_orders"] + exact: CompiledModelView = models["exact_orders"] assert exact.schema_entry is not None assert ( tuple(column.name for column in exact.schema_entry.columns) @@ -265,7 +267,7 @@ def test_given_inherited_model_schemas_when_compiling_then_attaches_resolved_con "nullable": False, } - superset: CompiledModel = models["order_superset"] + superset: CompiledModelView = models["order_superset"] assert superset.schema_entry is not None assert superset.schema_entry.description == "Canonical order schema" assert ( @@ -367,7 +369,7 @@ def test_given_named_schema_contract_mismatch_when_validating_then_reports_expec ) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - project: CompiledProject = assemble_project( + project: CompiledProjectView = assemble_project( inputs=build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, @@ -453,7 +455,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac """, }, ) - project: CompiledProject = assemble_project( + project: CompiledProjectView = assemble_project( inputs=build_compile_inputs( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, @@ -461,8 +463,8 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac ), inference_profile=DuckDbAdapter().expression_inference_profile(), ) - models_by_name: dict[str, CompiledModel] = {item.name: item for item in project.models} - model: CompiledModel = models_by_name["customer_category_amounts"] + models_by_name: dict[str, CompiledModelView] = {item.name: item for item in project.models} + model: CompiledModelView = models_by_name["customer_category_amounts"] assert model.schema_entry is not None assert tuple(column.name for column in model.inferred_columns or ()) == ( @@ -496,7 +498,7 @@ def test_given_dynamic_pivot_family_when_compiling_then_preserves_closed_contrac manifest_meta: dict[str, object] = cast(dict[str, object], manifest_node["meta"]) sqlbuild_meta: dict[str, object] = cast(dict[str, object], manifest_meta["sqlbuild"]) assert sqlbuild_meta["dynamic_columns"] == contract_identity["dynamic_columns"] - passthrough: CompiledModel = models_by_name["customer_category_amounts_copy"] + passthrough: CompiledModelView = models_by_name["customer_category_amounts_copy"] assert passthrough.dynamic_column_contract is not None assert passthrough.dynamic_column_contract.output_proven is True assert tuple(column.name for column in passthrough.inferred_columns or ()) == ("customer_id",) @@ -556,7 +558,7 @@ def test_given_partial_upstream_schema_when_compiling_dynamic_pivot_then_closure """, }, ) - project: CompiledProject = assemble_project( + project: CompiledProjectView = assemble_project( inputs=build_compile_inputs( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, @@ -644,7 +646,7 @@ def test_given_unenforced_dynamic_upstream_when_compiling_passthrough_then_closu """, }, ) - project: CompiledProject = assemble_project( + project: CompiledProjectView = assemble_project( inputs=build_compile_inputs( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, @@ -692,7 +694,7 @@ def test_given_parent_schema_type_change_when_fingerprinting_superset_then_ident } write_repo_files(tmp_path, repo_files) - initial: CompiledModel = compile_first_model(project_dir=tmp_path) + initial: CompiledModelView = compile_first_model(project_dir=tmp_path) write_repo_files( tmp_path, repo_files @@ -703,7 +705,7 @@ def test_given_parent_schema_type_change_when_fingerprinting_superset_then_ident """ }, ) - changed: CompiledModel = compile_first_model(project_dir=tmp_path) + changed: CompiledModelView = compile_first_model(project_dir=tmp_path) assert ( build_model_version_identity_metadata_json(model=initial) @@ -747,7 +749,7 @@ def test_given_local_cursor_column_when_validating_named_contract_then_combined_ }, ) - model: CompiledModel = compile_first_model(project_dir=tmp_path) + model: CompiledModelView = compile_first_model(project_dir=tmp_path) assert model.schema_entry is not None assert ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_namespace_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_namespace_validation.py deleted file mode 100644 index e02d7fcf4a..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_namespace_validation.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Tests for preserve target logical namespace validation.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile._helpers.config.namespace_validation import ( - validate_preserved_logical_namespace, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.spec.contracts.models import TargetConfig -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - PreservedLogicalNamespaceTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - PreservedLogicalNamespaceTestCase( - description="preserved schema without logical schema fails closed", - logical_database="analytics", - logical_schema=None, - target_database=None, - target_schema="preserve", - expected_error_fragment="Model 'orders' has no logical schema", - ), - PreservedLogicalNamespaceTestCase( - description="preserved database without logical database fails closed", - logical_database=None, - logical_schema="marts", - target_database="preserve", - target_schema=None, - expected_error_fragment="Model 'orders' has no logical database", - ), - PreservedLogicalNamespaceTestCase( - description="both preserved dimensions report one structured error", - logical_database=None, - logical_schema=None, - target_database="preserve", - target_schema="preserve", - expected_error_fragment="no logical database and schema", - ), - ), - ids=lambda case: case.description, -) -def test_given_missing_logical_namespace_when_validating_preserve_then_structured_error_is_raised( - test_case: PreservedLogicalNamespaceTestCase, -) -> None: - target: TargetConfig = TargetConfig( - database=test_case.target_database, - schema=test_case.target_schema, - ) - with pytest.raises(CompileInputError, match=str(test_case.expected_error_fragment)) as error: - validate_preserved_logical_namespace( - resource_label="Model 'orders'", - logical_database=test_case.logical_database, - logical_schema=test_case.logical_schema, - target_config=target, - ) - assert error.value.help is not None - assert "defaults" in error.value.help - - -@pytest.mark.parametrize( - "test_case", - ( - PreservedLogicalNamespaceTestCase( - description="literal target supplies namespace without logical values", - logical_database=None, - logical_schema=None, - target_database="physical_db", - target_schema="physical_schema", - expected_error_fragment=None, - ), - PreservedLogicalNamespaceTestCase( - description="missing target fields retain adapter fallback", - logical_database=None, - logical_schema=None, - target_database=None, - target_schema=None, - expected_error_fragment=None, - ), - ), - ids=lambda case: case.description, -) -def test_given_non_preserved_target_when_validating_namespace_then_adapter_fallback_remains_allowed( - test_case: PreservedLogicalNamespaceTestCase, -) -> None: - validate_preserved_logical_namespace( - resource_label="Model 'orders'", - logical_database=test_case.logical_database, - logical_schema=test_case.logical_schema, - target_config=TargetConfig( - database=test_case.target_database, - schema=test_case.target_schema, - ), - ) - assert test_case.expected_error_fragment is None diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py deleted file mode 100644 index ecbbe31173..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py +++ /dev/null @@ -1,275 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter -from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile._helpers.sql_tests.core import ( - extract_assertion_target_model_names, -) -from sqlbuild.compiler.compile._helpers.sql_tests.native import ( - extract_expanded_sql_tests, - extract_unexpanded_sql_test, -) -from sqlbuild.compiler.compile.models import CompileModelSqlTestCtes, CompileSqlTestCtes -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - DialectCteScanTestCase, - ExpectedMessageTestCase, - NativeSqlTestExtractionParityTestCase, -) - -_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() - - -@pytest.mark.parametrize( - "test_case", - ( - NativeSqlTestExtractionParityTestCase( - description="model roles retain authored ordering and independent checks", - sql=( - "WITH helper AS (SELECT 1 AS order_id), " - "__source__raw_orders AS (SELECT order_id FROM helper), " - "__macro__status AS (SELECT '''created'''), " - "__expected__orders AS (SELECT order_id FROM helper), " - "__assert__positive AS (SELECT order_id FROM helper WHERE order_id < 0) " - "SELECT 1" - ), - mode=SqlTestMode.MODEL, - ), - NativeSqlTestExtractionParityTestCase( - description="macro direct logic retains helper actual and expected payloads", - sql=( - "WITH input AS (SELECT 'created' AS status), " - "__macro_actual__ AS (SELECT @normalize_status(status) AS status FROM input), " - "__macro_expected__ AS (SELECT 'created' AS status) SELECT 1" - ), - mode=SqlTestMode.MACRO, - ), - NativeSqlTestExtractionParityTestCase( - description="udf direct logic retains helper actual and expected payloads", - sql=( - "WITH input AS (SELECT 1 AS value), " - '__udf_actual__ AS (SELECT __udf("increment")(value) AS value FROM input), ' - "__udf_expected__ AS (SELECT 2 AS value) SELECT 1" - ), - mode=SqlTestMode.UDF, - ), - NativeSqlTestExtractionParityTestCase( - description="table function direct logic retains actual and expected payloads", - sql=( - "__table_fn_actual__ AS (SELECT order_id FROM " - '__table_fn("customer_orders")(1)), ' - "__table_fn_expected__ AS (SELECT 1 AS order_id) SELECT 1" - ).join(("WITH ", "")), - mode=SqlTestMode.TABLE_FN, - ), - NativeSqlTestExtractionParityTestCase( - description="expected projection before where without from is accepted", - sql=( - "WITH __source__raw_orders AS (SELECT 1 AS id), " - "__expected__orders AS (" - "SELECT CAST(NULL AS INTEGER) AS id WHERE 1 = 0" - ") SELECT 1" - ), - mode=SqlTestMode.MODEL, - ), - NativeSqlTestExtractionParityTestCase( - description="qualified expected projection uses the column name", - sql=( - "WITH __source__raw_orders AS (SELECT 1 AS id), " - "sample AS (SELECT 1 AS id), " - "__expected__orders AS (SELECT sample.id FROM sample) SELECT 1" - ), - mode=SqlTestMode.MODEL, - ), - NativeSqlTestExtractionParityTestCase( - description="clause-like qualified column uses its explicit alias", - sql=( - "WITH __source__raw_orders AS (SELECT 1 AS id), " - "sample AS (SELECT 1 AS limit), " - "__expected__orders AS (SELECT sample.limit AS id FROM sample) SELECT 1" - ), - mode=SqlTestMode.MODEL, - ), - ), - ids=lambda case: case.description, -) -def test_given_macro_free_sql_tests_when_extracting_before_and_after_expansion_then_payloads_match( - test_case: NativeSqlTestExtractionParityTestCase, -) -> None: - expanded: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( - tests=((test_case.sql, "tests/unit/example.sql", test_case.mode),), - syntax=_GENERIC_SQL_SYNTAX, - ) - unexpanded: tuple[CompileSqlTestCtes, bool] = extract_unexpanded_sql_test( - sql=test_case.sql, - file_label="tests/unit/example.sql", - mode=test_case.mode, - syntax=_GENERIC_SQL_SYNTAX, - ) - - assert (expanded == (unexpanded[0],), unexpanded[1]) == (test_case.expected_matches, False) - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedMessageTestCase( - description="a check reading an expected CTE through a helper", - value=( - "WITH __source__raw_orders AS (SELECT 1 AS order_id), " - "__expected__orders AS (SELECT 1 AS order_id), " - "helper AS (SELECT order_id FROM __expected__orders), " - "__assert__same AS (SELECT order_id FROM helper) SELECT 1" - ), - expected_message=( - "SQL test 'tests/unit/example.sql' check CTE '__assert__same' must not depend on " - "'__expected__orders' through 'helper'; expected results and assertions must be " - "independent" - ), - ), - ExpectedMessageTestCase( - description="a check joining an expected CTE in a comma-separated FROM", - value=( - "WITH __source__orders AS (SELECT 1 AS id), " - "__expected__orders AS (SELECT 1 AS id), " - "__assert__valid AS (" - "SELECT expected.id FROM __source__orders source, __expected__orders expected" - ") SELECT 1" - ), - expected_message=( - "SQL test 'tests/unit/example.sql' check CTE '__assert__valid' must not depend on " - "'__expected__orders'; expected results and assertions must be independent" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_cross_check_dependency_when_extracting_then_independence_error_is_raised( - test_case: ExpectedMessageTestCase, -) -> None: - with pytest.raises(ValueError) as native_error: - extract_expanded_sql_tests( - tests=((test_case.value, "tests/unit/example.sql", SqlTestMode.MODEL),), - syntax=_GENERIC_SQL_SYNTAX, - ) - - assert str(native_error.value) == test_case.expected_message - - -@pytest.mark.parametrize( - "test_case", - [ - DialectCteScanTestCase( - description="snowflake backslash-escaped quote before a marker", - syntax=SnowflakeAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__ref__stg_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" - "__assert__named AS (\n" - " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" - ")\n" - "SELECT 1" - ), - expected_cte_names=("__ref__stg_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - DialectCteScanTestCase( - description="bigquery hash comment and triple quotes around markers", - syntax=BigQueryAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__ref__stg_orders AS (SELECT '''it's ), (''' AS note), # ), (\n" - "__assert__named AS (\n" - ' SELECT * FROM __ref("order_totals") # __ref("ignored")\n' - ")\n" - "SELECT 1" - ), - expected_cte_names=("__ref__stg_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - DialectCteScanTestCase( - description="databricks backslash-escaped quote before a marker", - syntax=DatabricksAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__ref__stg_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" - "__assert__named AS (\n" - " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" - ")\n" - "SELECT 1" - ), - expected_cte_names=("__ref__stg_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - DialectCteScanTestCase( - description="duckdb escape string and nested comment before a marker", - syntax=DuckDbAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__ref__stg_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" - "/* outer /* inner */ ), ( */\n" - "__assert__named AS (\n" - " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" - ")\n" - "SELECT 1" - ), - expected_cte_names=("__ref__stg_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - DialectCteScanTestCase( - description="postgres escape string before a marker", - syntax=PostgresAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__ref__stg_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" - "__assert__named AS (\n" - " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" - ")\n" - "SELECT 1" - ), - expected_cte_names=("__ref__stg_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - ], - ids=lambda case: case.description, -) -def test_given_dialect_sql_test_when_batch_extracting_then_follows_dialect_lexical_rules( - test_case: DialectCteScanTestCase, -) -> None: - extracted: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( - tests=( - (test_case.sql, "tests/unit/orders.sql", SqlTestMode.MODEL), - ( - "WITH __ref__a AS (SELECT 1 AS id), __expected__b AS (SELECT 1 AS id) SELECT 1", - "tests/unit/plain.sql", - SqlTestMode.MODEL, - ), - ), - syntax=test_case.syntax, - ) - - payload: object = extracted[0].payload - assert isinstance(payload, CompileModelSqlTestCtes) - assert ( - tuple(cte.name for cte in (*payload.authored_ctes, *payload.assertion_ctes)) - == test_case.expected_cte_names - ) - assert ( - extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in payload.assertion_ctes), - syntax=test_case.syntax, - ) - == test_case.expected_assertion_targets - ) - assert isinstance(extracted[1].payload, CompileModelSqlTestCtes) - assert extracted[1].payload.expected_model_names == ("b",) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 2afc430668..359efd5874 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -3,22 +3,23 @@ from collections.abc import Callable from decimal import Decimal from pathlib import Path -from unittest.mock import Mock import pytest from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile._helpers.assembly.project import assemble_compiled_project -from sqlbuild.compiler.compile._helpers.render.parameters import expand_test_parameters from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, CompiledSqlTest, - CompileProjectInputs, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, TypedSqlValueRenderer +from sqlbuild.compiler.compile.types import ( + CompiledProjectView, + CompiledResourceType, + CompileInputsView, + TypedSqlValueRenderer, +) from sqlbuild.compiler.planner._helpers.output.plan_entry import scope_overlaps from sqlbuild.sql_values.models import SqlLogicalType, SqlValue from sqlbuild.sql_values.types import SqlValueKind @@ -245,8 +246,8 @@ def test_given_parameterized_sql_tests_when_compiling_then_cases_have_typed_uniq write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: write_repo_files(tmp_path, test_case.repo_files) - compile_inputs: CompileProjectInputs = compile_project_inputs(project_dir=tmp_path) - compiled: CompiledProject = assemble_compiled_project(inputs=compile_inputs) + compile_inputs: CompileInputsView = compile_project_inputs(project_dir=tmp_path) + compiled: CompiledProjectView = assemble_compiled_project(inputs=compile_inputs) assert tuple(test.name for test in compiled.sql_tests) == test_case.expected_names assert tuple(test.key.name for test in compiled.sql_tests) == test_case.expected_names @@ -288,7 +289,7 @@ def test_given_parameterized_sql_tests_when_compiling_then_cases_have_typed_uniq model_test_path.read_text(encoding="utf-8").replace('amount "2.4700"', 'amount "2.4800"'), encoding="utf-8", ) - changed: CompiledProject = assemble_compiled_project( + changed: CompiledProjectView = assemble_compiled_project( inputs=compile_project_inputs(project_dir=tmp_path) ) changed_fingerprints: tuple[str | None, ...] = tuple( @@ -358,17 +359,9 @@ def test_given_adapter_specific_parameter_when_expanding_then_adapter_literal_is "sqlserver": SqlServerAdapter(), }[test_case.adapter_name] - rendered, used_names = expand_test_parameters( - sql='SELECT @param("value")', - file_path=Path("tests/unit/adapter.sql"), - values=(("value", test_case.value),), - value_renderer=renderer, - test_name="adapter rendering", - case_name="one", - ) + rendered: str = renderer.render_typed_scalar(value=test_case.value) - assert rendered == test_case.expected_sql - assert used_names == frozenset(("value",)) + assert f"SELECT {rendered}" == test_case.expected_sql @pytest.mark.parametrize( @@ -417,32 +410,37 @@ def test_given_invalid_parameter_references_when_compiling_then_clear_error_is_r [ ParameterizedSqlTestRenderingErrorTestCase( description="reports adapter test case and parameter when rendered value is oversized", - rendered_value="x" * 1_000_001, + rendered_value="'" * 600_000, expected_error_fragment=( - "SQL test 'large values' case 'oversized' parameter 'value' could not be " - "rendered by adapter 'test_adapter'" + "SQL test 'large_values' case 'oversized' parameter 'value' could not be " + "rendered by adapter 'duckdb': SQL test 'large_values' case 'oversized' " + "parameter 'value' rendered value is 1200002 bytes; maximum is 1000000 bytes" ), ) ], ids=lambda case: case.description, ) -def test_given_oversized_rendered_parameter_when_expanding_then_contextual_error_is_raised( +def test_given_oversized_rendered_parameter_when_compiling_then_contextual_error_is_raised( test_case: ParameterizedSqlTestRenderingErrorTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: - renderer: Mock = Mock() - renderer.adapter_name = "test_adapter" - renderer.render_typed_scalar.return_value = test_case.rendered_value - value: SqlValue = SqlValue( - logical_type=SqlLogicalType(SqlValueKind.STRING), - value="safe", + write_repo_files( + tmp_path, + base_repo_files() + | { + "tests/unit/large_values.sql": ( + "TEST (parameters (value string), cases (oversized (value " + f'"{test_case.rendered_value}")));\n\nSELECT @param("value")\n' + ) + }, ) - with pytest.raises(CompileInputError, match=test_case.expected_error_fragment): - expand_test_parameters( - sql='SELECT @param("value")', - file_path=Path("tests/unit/large.sql"), - values=(("value", value),), - value_renderer=renderer, - test_name="large values", - case_name="oversized", - ) + with pytest.raises(CompileInputError) as error: + compile_project_inputs(project_dir=tmp_path) + + assert str(error.value) == test_case.expected_error_fragment + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py index 85a6f1f44c..848cb0a319 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_python_sql_references.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( PythonSqlReferenceAllowedTestCase, PythonSqlReferenceErrorTestCase, @@ -366,7 +366,7 @@ def test_given_explicit_or_non_project_relations_when_compiling_then_it_succeeds ) -> None: write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files) - project: CompiledProject = compile_and_assemble(project_dir=tmp_path) + project: CompiledProjectView = compile_and_assemble(project_dir=tmp_path) assert tuple(sorted(model.name for model in project.models)) == test_case.expected_model_names @@ -395,7 +395,7 @@ def test_given_enforcement_disabled_when_compiling_hard_coded_names_then_it_succ ) -> None: write_repo_files(tmp_path, {"sqlbuild_project.toml": _DISABLED_PROJECT_FILE} | test_case.files) - project: CompiledProject = compile_and_assemble(project_dir=tmp_path) + project: CompiledProjectView = compile_and_assemble(project_dir=tmp_path) assert tuple(sorted(model.name for model in project.models)) == test_case.expected_model_names diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py deleted file mode 100644 index b1a145bfa0..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py +++ /dev/null @@ -1,691 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter -from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter -from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.compile._helpers.attachment.sql_tests import build_scenario_inputs -from sqlbuild.compiler.compile._helpers.scenarios.core import ( - extract_sql_scenario_ctes, - extract_sql_scenario_expected_model_names, -) -from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_assertion_target_model_names -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileSqlScenarioCtes, - CompileSqlScenarioInput, - MacroContext, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSourceFile, - DiscoveredSqlScenarioFile, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - BuildScenarioInputsErrorTestCase, - BuildScenarioInputsTestCase, - CteScannerMessageTestCase, - DialectCteScanTestCase, - ExtractSqlScenarioCtesErrorTestCase, - ExtractSqlScenarioCtesTestCase, -) -from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( - DUCKDB_DECLARATION_EXPANSION_CONTEXT, -) - -_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() - - -@pytest.mark.parametrize( - "test_case", - [ - ExtractSqlScenarioCtesTestCase( - description="extracts dbt ref fixtures", - sql=""" - WITH - __dbt_ref__orders AS (SELECT 1 AS order_id), - __dbt_ref__stripe__payments AS (SELECT 1 AS payment_id), - __expected__fact_orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__dbt_ref__orders", "__dbt_ref__stripe__payments"), - expected_source_fixture_names=(), - expected_ref_fixture_names=(), - expected_dbt_ref_fixture_names=("orders", "stripe__payments"), - expected_expected_model_names=("fact_orders",), - expected_assertion_names=(), - ), - ExtractSqlScenarioCtesTestCase( - description="extracts fixtures when the trailing select is omitted", - sql=""" - WITH - __ref__stg_orders AS (SELECT 1 AS order_id), - __expected__daily_revenue AS (SELECT 1 AS order_id) - """.strip(), - expected_authored_cte_names=("__ref__stg_orders",), - expected_source_fixture_names=(), - expected_ref_fixture_names=("stg_orders",), - expected_dbt_ref_fixture_names=(), - expected_expected_model_names=("daily_revenue",), - expected_assertion_names=(), - ), - ExtractSqlScenarioCtesTestCase( - description="extracts source ref seed fixtures expectations and assertions", - sql=""" - WITH - helper_customers AS (SELECT 10 AS customer_id), - __source__raw__orders AS (SELECT 1 AS order_id, 10 AS customer_id), - __ref__stg_customers AS (SELECT customer_id FROM helper_customers), - __seed__waffle_types AS (SELECT 1 AS waffle_type_id), - __expected__daily_revenue AS (SELECT 1 AS order_id), - __assert__no_negative_revenue AS ( - SELECT * FROM __ref("daily_revenue") WHERE revenue < 0 - ) - SELECT 1 - """.strip(), - expected_authored_cte_names=( - "helper_customers", - "__source__raw__orders", - "__ref__stg_customers", - "__seed__waffle_types", - ), - expected_source_fixture_names=("raw__orders",), - expected_ref_fixture_names=("stg_customers",), - expected_seed_fixture_names=("waffle_types",), - expected_expected_model_names=("daily_revenue",), - expected_assertion_names=("no_negative_revenue",), - ), - ExtractSqlScenarioCtesTestCase( - description="allows assertion only scenario target inference later", - sql=""" - WITH - __source__raw__orders AS (SELECT 1 AS order_id), - __assert__has_no_null_orders AS ( - SELECT * FROM __ref("fact_orders") WHERE order_id IS NULL - ) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__source__raw__orders",), - expected_source_fixture_names=("raw__orders",), - expected_ref_fixture_names=(), - expected_expected_model_names=(), - expected_assertion_names=("has_no_null_orders",), - ), - ExtractSqlScenarioCtesTestCase( - description="allows expected only scenario with ref fixture", - sql=""" - WITH - __ref__orders_base AS (SELECT 1 AS order_id), - __expected__fact_orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__ref__orders_base",), - expected_source_fixture_names=(), - expected_ref_fixture_names=("orders_base",), - expected_expected_model_names=("fact_orders",), - expected_assertion_names=(), - ), - ExtractSqlScenarioCtesTestCase( - description="allows expected only scenario with seed fixture", - sql=""" - WITH - __seed__country_codes AS (SELECT 'US' AS country_code), - __expected__dim_countries AS (SELECT 'US' AS country_code) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__seed__country_codes",), - expected_source_fixture_names=(), - expected_ref_fixture_names=(), - expected_seed_fixture_names=("country_codes",), - expected_expected_model_names=("dim_countries",), - expected_assertion_names=(), - ), - ], - ids=lambda case: case.description, -) -def test_given_sql_scenario_cte_variants_when_extracting_then_it_returns_expected_roles( - test_case: ExtractSqlScenarioCtesTestCase, -) -> None: - extracted_ctes: CompileSqlScenarioCtes = extract_sql_scenario_ctes( - syntax=_GENERIC_SQL_SYNTAX, - sql=test_case.sql, - file_label="tests/scenarios/revenue__customer_refund.sql", - ) - - assert ( - tuple(cte.name for cte in extracted_ctes.authored_ctes) - == test_case.expected_authored_cte_names - ) - assert extracted_ctes.source_fixture_names == test_case.expected_source_fixture_names - assert extracted_ctes.ref_fixture_names == test_case.expected_ref_fixture_names - assert extracted_ctes.seed_fixture_names == test_case.expected_seed_fixture_names - assert extracted_ctes.dbt_ref_fixture_names == test_case.expected_dbt_ref_fixture_names - assert extracted_ctes.expected_model_names == test_case.expected_expected_model_names - assert extracted_ctes.assertion_names == test_case.expected_assertion_names - - -@pytest.mark.parametrize( - "test_case", - [ - ExtractSqlScenarioCtesErrorTestCase( - description="raises when ceremonial select is wrapped in a final result cte", - sql=""" - WITH - __ref__orders_base AS (SELECT 1 AS order_id), - __expected__fact_orders AS (SELECT 1 AS order_id), - scenario_result AS (SELECT 1) - SELECT * FROM scenario_result - """.strip(), - expected_error_fragment="must end after its CTEs", - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when assertion depends directly on expected result", - sql=""" - WITH - __source__raw__orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id), - __assert__expected_is_nonempty AS (SELECT * FROM __expected__orders) - SELECT 1 - """.strip(), - expected_error_fragment=( - "'__assert__expected_is_nonempty' must not depend on '__expected__orders'" - ), - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when expected result depends indirectly on assertion", - sql=""" - WITH - __source__raw__orders AS (SELECT 1 AS order_id), - __assert__positive_order_id AS (SELECT 1 AS order_id WHERE order_id < 1), - assertion_helper AS (SELECT * FROM __assert__positive_order_id), - __expected__orders AS (SELECT * FROM assertion_helper) - SELECT 1 - """.strip(), - expected_error_fragment=( - "'__expected__orders' must not depend on " - "'__assert__positive_order_id' through 'assertion_helper'" - ), - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when a scenario cte name is quoted", - sql=""" - WITH - "__source__raw__orders" AS (SELECT 1 AS order_id), - __expected__daily_revenue AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment="expected a CTE name", - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when a scenario cte uses a materialization hint", - sql=""" - WITH - __source__raw__orders AS MATERIALIZED (SELECT 1 AS order_id), - __expected__daily_revenue AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment=( - "CTE '__source__raw__orders' must not use AS MATERIALIZED; materialization " - "hints are not supported in SQL scenario CTEs" - ), - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when scenario has no fixture ctes", - sql=""" - WITH __expected__fact_orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment=( - r"must define at least one __source__\*, __ref__\*, __seed__\*, " - r"or __dbt_ref__\* fixture CTE" - ), - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when scenario has no expected or assertion ctes", - sql=""" - WITH __source__raw__orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment=( - "must define at least one __expected__ or __assert__" - ), - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when scenario uses macro mock cte", - sql=""" - WITH - __source__raw__orders AS (SELECT 1 AS order_id), - __macro__country AS (SELECT '''US'''), - __expected__fact_orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment="does not support macro mock CTE '__macro__country'", - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when assertion cte suffix is missing", - sql=""" - WITH - __source__raw__orders AS (SELECT 1 AS order_id), - __assert__ AS (SELECT * FROM __ref("fact_orders")) - SELECT 1 - """.strip(), - expected_error_fragment="__assert__", - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when duplicate cte names are present", - sql=""" - WITH - __source__raw__orders AS (SELECT 1 AS order_id), - __source__raw__orders AS (SELECT 2 AS order_id), - __expected__fact_orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment="defines duplicate CTE '__source__raw__orders'", - ), - ExtractSqlScenarioCtesErrorTestCase( - description="raises when fallback terminal read follows a set operation", - sql=""" - WITH - "__ref__orders_base" AS (SELECT 1 AS order_id), - "__expected__fact_orders" AS (SELECT 1 AS order_id), - scenario_result AS (SELECT 1) - SELECT 2 UNION ALL SELECT * FROM scenario_result - """.strip(), - expected_error_fragment="expected a CTE name", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_sql_scenario_ctes_when_extracting_then_it_raises_clear_errors( - test_case: ExtractSqlScenarioCtesErrorTestCase, -) -> None: - with pytest.raises(ValueError, match=test_case.expected_error_fragment): - extract_sql_scenario_ctes( - syntax=_GENERIC_SQL_SYNTAX, - sql=test_case.sql, - file_label="tests/scenarios/revenue__customer_refund.sql", - ) - - -@pytest.mark.parametrize( - "test_case", - [ - BuildScenarioInputsTestCase( - description="attaches scenario cte roles after authored sql expansion", - sql_body=""" - WITH - __source__raw__orders AS (SELECT @@customer_id AS customer_id), - __expected__daily_revenue AS (SELECT @@customer_id AS customer_id) - SELECT 1 - """.strip(), - effective_vars={"customer_id": "10"}, - expected_source_fixture_names=("raw__orders",), - expected_ref_fixture_names=(), - expected_seed_fixture_names=(), - expected_expected_model_names=("daily_revenue",), - expected_assertion_names=(), - expected_sql_fragment="SELECT 10 AS customer_id", - ), - BuildScenarioInputsTestCase( - description="attaches seed only scenario fixture roles", - sql_body=""" - WITH - __seed__country_codes AS (SELECT '@@country_code' AS country_code), - __expected__dim_countries AS (SELECT '@@country_code' AS country_code) - SELECT 1 - """.strip(), - effective_vars={"country_code": "US"}, - expected_source_fixture_names=(), - expected_ref_fixture_names=(), - expected_seed_fixture_names=("country_codes",), - expected_expected_model_names=("dim_countries",), - expected_assertion_names=(), - expected_sql_fragment="SELECT 'US' AS country_code", - ), - BuildScenarioInputsTestCase( - description="allows project source references in scenario fixture ctes", - sql_body=""" - WITH - __source__raw__orders AS ( - SELECT order_id FROM __source("raw__orders") WHERE order_id <= 10 - ), - __expected__daily_revenue AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - effective_vars={}, - expected_source_fixture_names=("raw__orders",), - expected_ref_fixture_names=(), - expected_seed_fixture_names=(), - expected_expected_model_names=("daily_revenue",), - expected_assertion_names=(), - expected_sql_fragment='__source("raw__orders")', - ), - ], - ids=lambda case: case.description, -) -def test_given_discovered_scenario_when_building_scenario_inputs_then_it_attaches_cte_roles( - test_case: BuildScenarioInputsTestCase, -) -> None: - scenario_file: DiscoveredSqlScenarioFile = DiscoveredSqlScenarioFile( - file_path=Path(__file__), - relative_path=Path("tests/scenarios/revenue__customer_refund.sql"), - contents="", - header_values={}, - name="revenue__customer_refund", - sql_body=test_case.sql_body, - ) - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig(name="demo", adapter="duckdb"), - local_config=LocalConfig(), - scenario_files=(scenario_file,), - source_files=( - DiscoveredSourceFile( - file_path=Path("sources/raw.yml"), - relative_path=Path("sources/raw.yml"), - contents="", - source_entries=(SourceEntry(name="raw__orders", schema="raw", table="orders"),), - ), - ), - ) - - scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( - sql_lexical_syntax=_GENERIC_SQL_SYNTAX, - discovered_inputs=discovered_inputs, - effective_vars=test_case.effective_vars, - macro_context=MacroContext( - adapter_name="duckdb", sql_analysis_enabled=True, target_name=None - ), - loaded_macros={}, - declaration_expansion=DUCKDB_DECLARATION_EXPANSION_CONTEXT, - ) - - assert len(scenario_inputs) == 1 - scenario_input: CompileSqlScenarioInput = scenario_inputs[0] - assert scenario_input.scenario_file.name == "revenue__customer_refund" - assert scenario_input.source_fixture_names == test_case.expected_source_fixture_names - assert scenario_input.ref_fixture_names == test_case.expected_ref_fixture_names - assert scenario_input.seed_fixture_names == test_case.expected_seed_fixture_names - assert scenario_input.dbt_ref_fixture_names == test_case.expected_dbt_ref_fixture_names - assert scenario_input.expected_model_names == test_case.expected_expected_model_names - assert scenario_input.assertion_names == test_case.expected_assertion_names - assert test_case.expected_sql_fragment in scenario_input.sql_body - - -@pytest.mark.parametrize( - "test_case", - [ - BuildScenarioInputsErrorTestCase( - description="rejects project source references in expected ctes", - sql_body=""" - WITH - __source__raw__orders AS (SELECT 1 AS order_id), - __expected__daily_revenue AS (SELECT order_id FROM __source("raw__orders")) - SELECT 1 - """.strip(), - expected_error_fragment="__expected__daily_revenue.*must not reference project source", - ), - BuildScenarioInputsErrorTestCase( - description="rejects unknown project source references in fixture ctes", - sql_body=""" - WITH - __source__raw__orders AS (SELECT order_id FROM __source("missing_source")), - __expected__daily_revenue AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment="references unknown source 'missing_source'", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_scenario_source_refs_when_building_inputs_then_it_raises_clear_error( - test_case: BuildScenarioInputsErrorTestCase, -) -> None: - scenario_file: DiscoveredSqlScenarioFile = DiscoveredSqlScenarioFile( - file_path=Path(__file__), - relative_path=Path("tests/scenarios/revenue__customer_refund.sql"), - contents="", - header_values={}, - name="revenue__customer_refund", - sql_body=test_case.sql_body, - ) - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig(name="demo", adapter="duckdb"), - local_config=LocalConfig(), - scenario_files=(scenario_file,), - source_files=( - DiscoveredSourceFile( - file_path=Path("sources/raw.yml"), - relative_path=Path("sources/raw.yml"), - contents="", - source_entries=(SourceEntry(name="raw__orders", schema="raw", table="orders"),), - ), - ), - ) - - with pytest.raises(ValueError, match=test_case.expected_error_fragment): - build_scenario_inputs( - sql_lexical_syntax=_GENERIC_SQL_SYNTAX, - discovered_inputs=discovered_inputs, - effective_vars={}, - macro_context=MacroContext( - adapter_name="duckdb", sql_analysis_enabled=True, target_name=None - ), - loaded_macros={}, - declaration_expansion=DUCKDB_DECLARATION_EXPANSION_CONTEXT, - ) - - -@pytest.mark.parametrize( - "test_case", - ( - CteScannerMessageTestCase( - description="scenario without a with clause uses scenario wording", - sql="SELECT * FROM orders", - expected_message=( - "SQL scenario 'tests/scenarios/orders.sql' must declare fixture CTEs and at least " - "one __expected__ or __assert__ CTE in a top-level WITH clause" - ), - ), - CteScannerMessageTestCase( - description="unclosed comment before a fixture uses scenario wording", - sql="WITH /* unfinished", - expected_message="SQL scenario contains an unclosed block comment", - ), - CteScannerMessageTestCase( - description="unclosed trailing comment uses scenario wording", - sql="WITH __expected__orders AS (SELECT 1 AS order_id) SELECT 1; /* unfinished", - expected_message="SQL scenario contains an unclosed block comment", - ), - CteScannerMessageTestCase( - description="scenario with an invalid cte name uses scenario wording", - sql="WITH 1orders AS (SELECT 1) SELECT 1", - expected_message="SQL scenario 'tests/scenarios/orders.sql' expected a CTE name", - ), - CteScannerMessageTestCase( - description="scenario without a ceremonial select uses scenario wording", - sql=( - "WITH __source__raw_orders AS (SELECT 1 AS order_id), " - "__expected__orders AS (SELECT 1 AS order_id), " - "result AS (SELECT 1) SELECT * FROM result" - ), - expected_message=( - "SQL scenario 'tests/scenarios/orders.sql' must end after its CTEs; only an optional " - "ceremonial top-level `SELECT 1` may follow them" - ), - ), - CteScannerMessageTestCase( - description="scenario fixture without a target uses scenario wording", - sql=( - "WITH __source__ AS (SELECT 1 AS order_id), " - "__expected__orders AS (SELECT 1 AS order_id) SELECT 1" - ), - expected_message=( - "SQL scenario 'tests/scenarios/orders.sql' must use __source__ to " - "identify a target" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_malformed_scenario_ctes_when_extracting_then_messages_name_the_scenario( - test_case: CteScannerMessageTestCase, -) -> None: - with pytest.raises(CompileInputError) as error_info: - _ = extract_sql_scenario_ctes( - sql=test_case.sql, file_label="tests/scenarios/orders.sql", syntax=_GENERIC_SQL_SYNTAX - ) - - assert str(error_info.value) == test_case.expected_message - - -@pytest.mark.parametrize( - "test_case", - ( - CteScannerMessageTestCase( - description="expected model scan without AS uses scenario wording", - sql="WITH __source__raw_orders (SELECT 1 AS order_id) SELECT 1", - expected_message="SQL scenario 'tests/scenarios/orders.sql' expected keyword AS", - ), - CteScannerMessageTestCase( - description="expected model scan unclosed comment uses scenario wording", - sql="WITH /* unfinished", - expected_message="SQL scenario contains an unclosed block comment", - ), - CteScannerMessageTestCase( - description="expected model scan leading unclosed comment uses scenario wording", - sql="/* unfinished", - expected_message="SQL scenario contains an unclosed block comment", - ), - CteScannerMessageTestCase( - description="expected model scan without a target uses scenario wording", - sql=( - "WITH __source__raw_orders AS (SELECT 1 AS order_id), " - "__expected__ AS (SELECT 1 AS order_id) SELECT 1" - ), - expected_message=( - "SQL scenario 'tests/scenarios/orders.sql' must use __expected__ to " - "identify a target" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_malformed_scenario_ctes_when_scanning_expected_models_then_names_the_scenario( - test_case: CteScannerMessageTestCase, -) -> None: - with pytest.raises(CompileInputError) as error_info: - _ = extract_sql_scenario_expected_model_names( - sql=test_case.sql, - file_label="tests/scenarios/orders.sql", - syntax=_GENERIC_SQL_SYNTAX, - ) - - assert str(error_info.value) == test_case.expected_message - - -@pytest.mark.parametrize( - "test_case", - [ - DialectCteScanTestCase( - description="snowflake backslash-escaped quote before a marker", - syntax=SnowflakeAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__source__raw_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" - "__assert__named AS (\n" - " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" - ")\n" - "SELECT 1" - ), - expected_cte_names=("__source__raw_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - DialectCteScanTestCase( - description="bigquery hash comment and triple quotes around markers", - syntax=BigQueryAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__source__raw_orders AS (SELECT '''it's ), (''' AS note), # ), (\n" - "__assert__named AS (\n" - ' SELECT * FROM __ref("order_totals") # __ref("ignored")\n' - ")\n" - "SELECT 1" - ), - expected_cte_names=("__source__raw_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - DialectCteScanTestCase( - description="databricks backslash-escaped quote before a marker", - syntax=DatabricksAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__source__raw_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" - "__assert__named AS (\n" - " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" - ")\n" - "SELECT 1" - ), - expected_cte_names=("__source__raw_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - DialectCteScanTestCase( - description="duckdb escape string and nested comment before a marker", - syntax=DuckDbAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__source__raw_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" - "/* outer /* inner */ ), ( */\n" - "__assert__named AS (\n" - " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" - ")\n" - "SELECT 1" - ), - expected_cte_names=("__source__raw_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - DialectCteScanTestCase( - description="postgres escape string before a marker", - syntax=PostgresAdapter.sql_lexical_syntax, - sql=( - "WITH\n" - "__source__raw_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" - "__assert__named AS (\n" - " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" - ")\n" - "SELECT 1" - ), - expected_cte_names=("__source__raw_orders", "__assert__named"), - expected_assertion_targets=("order_totals",), - ), - ], - ids=lambda case: case.description, -) -def test_given_dialect_scenario_sql_when_extracting_ctes_then_follows_dialect_lexical_rules( - test_case: DialectCteScanTestCase, -) -> None: - extracted: CompileSqlScenarioCtes = extract_sql_scenario_ctes( - sql=test_case.sql, file_label="tests/scenarios/orders.sql", syntax=test_case.syntax - ) - - assert ( - tuple(cte.name for cte in (*extracted.authored_ctes, *extracted.assertion_ctes)) - == test_case.expected_cte_names - ) - assert ( - extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in extracted.assertion_ctes), - syntax=test_case.syntax, - ) - == test_case.expected_assertion_targets - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py index 12f6f8acf7..0d074171cf 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scoped_named_declarations.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.scopes.types import DiagnosticSeverity from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( NamedDeclarationAcceptedTestCase, @@ -199,7 +199,7 @@ def test_given_valid_named_declaration_layout_when_compiling_then_project_is_acc ) -> None: write_repo_files(tmp_path, {"sqlbuild_project.toml": _PROJECT_FILE} | test_case.files) - compiled: CompiledProject = compile_and_assemble(project_dir=tmp_path) + compiled: CompiledProjectView = compile_and_assemble(project_dir=tmp_path) assert compiled.scope_index.completeness.placement is True assert ( @@ -556,7 +556,7 @@ def test_given_advisory_placement_when_compiling_misplaced_declaration_then_warn ) -> None: write_repo_files(tmp_path, {"sqlbuild_project.toml": _ADVISORY_PROJECT_FILE} | test_case.files) - compiled: CompiledProject = compile_and_assemble(project_dir=tmp_path) + compiled: CompiledProjectView = compile_and_assemble(project_dir=tmp_path) assert tuple( (diagnostic.code.value, diagnostic.severity) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py deleted file mode 100644 index c12fa0d171..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Set-operation expected CTEs across every SQL-test kind validate every branch.""" - -from __future__ import annotations - -from itertools import product - -import pytest - -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlTestCtes -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - SetOperationExpectedKind, - SetOperationExpectedTestCase, -) -from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import extract_test_ctes - -_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() - -_FILE: str = "tests/unit/orders.sql" -_MISMATCH: str = ( - "SQL test '{file}' must use the same {cte} projection names and order in every " - "set-operation branch; branch {branch} does not match branch 1" -) -_NOT_SELECT: str = "SQL test '{file}' must define each {cte} set-operation branch as a SELECT query" -_KINDS: tuple[SetOperationExpectedKind, ...] = ( - SetOperationExpectedKind( - description="model", - mode=SqlTestMode.MODEL, - support_ctes=(("__source__raw_orders", "SELECT 1 AS order_id"),), - expected_cte_name="__expected__orders", - payload_type="CompileModelSqlTestCtes", - ), - SetOperationExpectedKind( - description="macro", - mode=SqlTestMode.MACRO, - support_ctes=(("__macro_actual__", "SELECT 1 AS order_id"),), - expected_cte_name="__macro_expected__", - payload_type="CompileDirectLogicSqlTestCtes", - ), - SetOperationExpectedKind( - description="udf", - mode=SqlTestMode.UDF, - support_ctes=(("__udf_actual__", "SELECT 1 AS order_id"),), - expected_cte_name="__udf_expected__", - payload_type="CompileDirectLogicSqlTestCtes", - ), - SetOperationExpectedKind( - description="table_fn", - mode=SqlTestMode.TABLE_FN, - support_ctes=(("__table_fn_actual__", "SELECT 1 AS order_id"),), - expected_cte_name="__table_fn_expected__", - payload_type="CompileDirectLogicSqlTestCtes", - ), -) -_ACCEPTED_SQL: tuple[str, ...] = ( - "SELECT 1 AS order_id INTERSECT SELECT 1 AS order_id", - "SELECT 1 AS order_id INTERSECT ALL SELECT 1 AS order_id", - "SELECT 1 AS order_id INTERSECT DISTINCT SELECT 1 AS order_id", - "SELECT 1 AS order_id EXCEPT SELECT 2 AS order_id", - "SELECT 1 AS order_id EXCEPT ALL SELECT 2 AS order_id", - "SELECT 1 AS order_id EXCEPT DISTINCT SELECT 2 AS order_id", - "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS order_id", -) -_REJECTED_SQL: tuple[tuple[str, str], ...] = ( - ("SELECT 1 AS order_id INTERSECT SELECT 1 AS other_id", _MISMATCH.replace("{branch}", "2")), - ( - "SELECT 1 AS order_id EXCEPT ALL SELECT 1 AS order_id, 2 AS extra", - _MISMATCH.replace("{branch}", "2"), - ), - ( - "SELECT 1 AS order_id UNION SELECT 2 AS order_id EXCEPT SELECT 3 AS other_id", - _MISMATCH.replace("{branch}", "3"), - ), - ("SELECT 1 AS order_id EXCEPT VALUES (1)", _NOT_SELECT), -) - - -@pytest.mark.parametrize( - "test_case", - [ - SetOperationExpectedTestCase( - description=f"{kind.description}: {sql}", - kind=kind, - sql=sql, - expected_payload_type=kind.payload_type, - ) - for kind, sql in product(_KINDS, _ACCEPTED_SQL) - ], - ids=lambda case: case.description, -) -def test_given_set_operation_expected_cte_when_classifying_then_it_is_accepted( - test_case: SetOperationExpectedTestCase, -) -> None: - ctes: tuple[tuple[str, str], ...] = ( - *test_case.kind.support_ctes, - (test_case.kind.expected_cte_name, test_case.sql), - ) - - classified: CompileSqlTestCtes = extract_test_ctes( - ctes=ctes, file_label=_FILE, mode=test_case.kind.mode - ) - - assert type(classified.payload).__name__ == test_case.expected_payload_type - - -@pytest.mark.parametrize( - "test_case", - [ - SetOperationExpectedTestCase( - description=f"{kind.description}: {rejected[0]}", - kind=kind, - sql=rejected[0], - expected_error_template=rejected[1], - ) - for kind, rejected in product(_KINDS, _REJECTED_SQL) - ], - ids=lambda case: case.description, -) -def test_given_invalid_set_operation_branch_when_classifying_then_it_names_the_expected_cte( - test_case: SetOperationExpectedTestCase, -) -> None: - ctes: tuple[tuple[str, str], ...] = ( - *test_case.kind.support_ctes, - (test_case.kind.expected_cte_name, test_case.sql), - ) - - with pytest.raises(CompileInputError) as error_info: - _ = extract_test_ctes(ctes=ctes, file_label=_FILE, mode=test_case.kind.mode) - - assert str(error_info.value) == test_case.expected_error_template.format( - file=_FILE, cte=test_case.kind.expected_cte_name - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_authored_offsets.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_authored_offsets.py deleted file mode 100644 index 40ce08355c..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_test_authored_offsets.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Offsets in a discovered SQL test body map back to the exact authored file position.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.compile._helpers.sql_tests.extraction_errors import authored_file_offset -from sqlbuild.compiler.discovery.main._parse_sql_test_file import parse_sql_test_file -from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import AuthoredOffsetTestCase - -_PATH: Path = Path("tests/unit/orders.sql") - - -@pytest.mark.parametrize( - "test_case", - [ - AuthoredOffsetTestCase( - description="flush-left body after a comment that repeats the marker", - contents="TEST ();\n-- zz_mark is named here\nWITH\nzz_mark AS (SELECT 1 AS a)\n", - marker="zz_mark AS", - expected_line_column=(4, 1), - ), - AuthoredOffsetTestCase( - description="indented body with a tab before the marker", - contents="TEST ();\n WITH\n h AS (SELECT 1 AS a),\n \tzz_mark AS (SELECT 1)\n", - marker="zz_mark", - expected_line_column=(4, 6), - ), - AuthoredOffsetTestCase( - description="tab inside a line before the marker", - contents="TEST ();\nWITH\nh AS (SELECT 1\t AS a, 'é' AS zz_mark)\n", - marker="zz_mark", - expected_line_column=(3, 30), - ), - AuthoredOffsetTestCase( - description="second block with blank lines after its header", - contents=( - 'TEST (name "one");\nWITH a AS (SELECT 1)\n;\nTEST (name "two");\n\n\n' - " WITH\n b AS (SELECT zz_mark)\n" - ), - marker="zz_mark", - expected_line_column=(8, 16), - ), - AuthoredOffsetTestCase( - description="first body line indented on the header line", - contents="TEST (); WITH zz_mark AS (SELECT 1)\n , b AS (SELECT 2)\n", - marker="zz_mark", - expected_line_column=(1, 17), - ), - ], - ids=lambda case: case.description, -) -def test_given_discovered_block_when_mapping_body_offset_then_points_at_authored_text( - test_case: AuthoredOffsetTestCase, -) -> None: - blocks: tuple[DiscoveredSqlTestBlock, ...] = parse_sql_test_file( - contents=test_case.contents, file_path=_PATH - ) - test_block: DiscoveredSqlTestBlock = blocks[-1] - test_file: DiscoveredSqlTestFile = DiscoveredSqlTestFile( - file_path=_PATH, relative_path=_PATH, contents=test_case.contents, blocks=blocks - ) - body_offset: int = test_block.sql_body.rindex(test_case.marker) - - offset: int | None = authored_file_offset( - test_file=test_file, test_block=test_block, body_offset=body_offset - ) - - contents: str = test_case.contents - resolved: int = offset or 0 - assert ( - contents[resolved : resolved + len(test_case.marker)], - contents.count("\n", 0, resolved) + 1, - resolved - contents.rfind("\n", 0, resolved), - ) == (test_case.marker, *test_case.expected_line_column) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py deleted file mode 100644 index 130ed2df68..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py +++ /dev/null @@ -1,1012 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile._helpers.sql_tests.core import ( - complete_omitted_ceremonial_select, - extract_sql_test_expected_model_names, -) -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileDirectLogicSqlTestCtes, - CompileModelSqlTestCtes, - CompileSqlTestCtes, -) -from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( - CompleteCeremonialSelectTestCase, - CteScannerMessageTestCase, - ExtractSqlTestCtesErrorTestCase, - ExtractSqlTestCtesTestCase, -) -from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import extract_test_sql - -_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() - - -@pytest.mark.parametrize( - "test_case", - [ - ExtractSqlTestCtesTestCase( - description="extracts dbt ref mocks from dbt ref prefixed ctes", - sql=""" - WITH - __dbt_ref__orders AS (SELECT 1 AS order_id), - __dbt_ref__stripe__payments AS (SELECT 1 AS payment_id), - __expected__fact_orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__dbt_ref__orders", "__dbt_ref__stripe__payments"), - expected_mock_model_names=(), - expected_mock_source_names=(), - expected_expected_model_names=("fact_orders",), - expected_mock_dbt_ref_names=("orders", "stripe__payments"), - ), - ExtractSqlTestCtesTestCase( - description="extracts model ctes when the trailing select is omitted", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id) - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - ), - ExtractSqlTestCtesTestCase( - description="extracts model ctes ending with a semicolon and comment", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id); - -- end of test - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - ), - ExtractSqlTestCtesTestCase( - description="extracts table function fixtures from model tests", - sql=""" - WITH - __table_fn__customer_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__table_fn__customer_orders",), - expected_mock_model_names=(), - expected_mock_source_names=(), - expected_expected_model_names=("orders",), - expected_mock_table_function_names=("customer_orders",), - ), - ExtractSqlTestCtesTestCase( - description="allows empty fixture marker for model expected output", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT * FROM __empty_fixture()) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - ), - ExtractSqlTestCtesTestCase( - description="extracts zero-row assertion ctes", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id, 10 AS amount), - __assert__no_negative_orders AS ( - SELECT * FROM __ref("orders") WHERE amount < 0 - ) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=(), - expected_assertion_names=("no_negative_orders",), - ), - ExtractSqlTestCtesTestCase( - description="extracts mixed expected and assertion ctes", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id, 10 AS amount), - __expected__orders AS (SELECT 1 AS order_id, 10 AS amount), - __assert__no_negative_orders AS ( - SELECT * FROM __ref("orders") WHERE amount < 0 - ) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - expected_assertion_names=("no_negative_orders",), - ), - ExtractSqlTestCtesTestCase( - description="allows expected result and assertion to share an independent helper", - sql=""" - WITH - helper AS (SELECT 1 AS order_id), - __source__raw_orders AS (SELECT order_id FROM helper), - __expected__orders AS (SELECT order_id FROM helper), - __assert__positive_order_id AS (SELECT order_id FROM helper WHERE order_id < 1) - SELECT 1 - """.strip(), - expected_authored_cte_names=("helper", "__source__raw_orders"), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - expected_assertion_names=("positive_order_id",), - ), - ExtractSqlTestCtesTestCase( - description="extracts seed mocks from seed-prefixed ctes", - sql=""" - WITH - __seed__country_codes AS (SELECT 'US' AS country_code), - __expected__orders AS (SELECT 'US' AS country_code) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__seed__country_codes",), - expected_mock_model_names=(), - expected_mock_source_names=(), - expected_expected_model_names=("orders",), - expected_mock_seed_names=("country_codes",), - ), - ExtractSqlTestCtesTestCase( - description="extracts macro mocks from single string literal ctes", - sql=""" - WITH - __macro__country AS (SELECT '''US'''), - __macro__empty AS (SELECT ''), - __macro__literal_text AS (SELECT ''' + x + '''), - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - expected_macro_mocks={ - "country": "'US'", - "empty": "", - "literal_text": "' + x + '", - }, - ), - ExtractSqlTestCtesTestCase( - description="extracts ctes from compact single line test sql", - sql=""" - WITH - helper AS (SELECT 1 AS order_id), - __ref__orders AS (SELECT * FROM helper), - __expected__order_items AS (SELECT order_id FROM __ref__orders) - SELECT 1; - """.strip(), - expected_authored_cte_names=("helper", "__ref__orders"), - expected_mock_model_names=("orders",), - expected_mock_source_names=(), - expected_expected_model_names=("order_items",), - ), - ExtractSqlTestCtesTestCase( - description="extracts ctes with comments strings and nested parentheses", - sql=""" - WITH /* leading */ helper_orders(order_id, note) AS ( - SELECT - CAST('value ) inside string' AS VARCHAR) AS note, - COALESCE(order_id, 0) AS order_id - FROM raw_orders -- comment with ) and , - ) , - __source__raw_orders AS (SELECT * FROM helper_orders), - __expected__orders AS ( - SELECT order_id, note FROM __source__raw_orders WHERE note != 'not ) done' - ) - -- final comment before ceremonial select - SELECT 1 - """.strip(), - expected_authored_cte_names=("helper_orders", "__source__raw_orders"), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - ), - ExtractSqlTestCtesTestCase( - description="extracts multiple expected targets with mixed whitespace", - sql=""" - WITH - __source__raw_orders AS(SELECT 1 AS order_id),__expected__orders AS( - SELECT 1 AS order_id - ), - __expected__daily_revenue AS (SELECT SUM(1) AS revenue) - SELECT 1; - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders", "daily_revenue"), - ), - ExtractSqlTestCtesTestCase( - description="extracts ctes through nested helper chains and awkward comma placement", - sql=""" - WITH source_seed(seed_id, payload) AS ( - SELECT * FROM ( - SELECT 1 AS seed_id, 'a,b,c' AS payload - UNION ALL - SELECT 2 AS seed_id, 'literal with ) and , chars' AS payload - ) AS nested_seed - )/* comma after block comment */, - normalized_orders AS( - SELECT seed_id AS order_id, CONCAT(payload, ')') AS note FROM source_seed - ) - ,__source__raw_orders AS( - SELECT * FROM normalized_orders WHERE note NOT IN ('x,y', 'z)') - ), expected_rows AS ( - SELECT order_id, note FROM __source__raw_orders - WHERE order_id IN (SELECT seed_id FROM source_seed WHERE seed_id > 0) - ), - __expected__orders AS (SELECT order_id, note FROM expected_rows) - SELECT 1; - """.strip(), - expected_authored_cte_names=( - "source_seed", - "normalized_orders", - "__source__raw_orders", - "expected_rows", - ), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - ), - ExtractSqlTestCtesTestCase( - description="extracts ctes with inline comments around separators", - sql=""" - WITH helper_a AS (SELECT 1 AS order_id) -- comment before comma - , helper_b AS (SELECT order_id FROM helper_a /* ) , */) - , __ref__orders AS (SELECT order_id FROM helper_b), -- trailing comma comment - __expected__orders AS ( - SELECT order_id FROM __ref__orders WHERE order_id = (SELECT MAX(order_id) FROM helper_b) - ) - SELECT 1 - """.strip(), - expected_authored_cte_names=("helper_a", "helper_b", "__ref__orders"), - expected_mock_model_names=("orders",), - expected_mock_source_names=(), - expected_expected_model_names=("orders",), - ), - ExtractSqlTestCtesTestCase( - description="allows explicit union branches in expected ctes", - sql=""" - WITH __source__raw_orders AS ( - SELECT 1 AS order_id, 'new' AS status - ), - __expected__orders AS ( - SELECT 1 AS order_id, 'new' AS status - UNION ALL - SELECT 2 AS order_id, 'paid' AS status - ) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - ), - ExtractSqlTestCtesTestCase( - description="allows plain union with matching aliases in expected ctes", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS ( - SELECT 1 AS order_id, 'created' AS status - UNION - SELECT 2 AS order_id, 'paid' AS status - ) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - ), - ExtractSqlTestCtesTestCase( - description="allows cast expressions with explicit aliases in expected ctes", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS ( - SELECT - CAST(1 AS INTEGER) AS order_id, - CAST('paid' AS VARCHAR) AS status - UNION ALL - SELECT - CAST(2 AS INTEGER) AS order_id, - CAST('created' AS VARCHAR) AS status - ) - SELECT 1 - """.strip(), - expected_authored_cte_names=("__source__raw_orders",), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - ), - ExtractSqlTestCtesTestCase( - description="allows bare identifiers in expected cte projections", - sql=""" - WITH expected_rows AS (SELECT 1 AS order_id, 'paid' AS status), - __source__raw_orders AS (SELECT * FROM expected_rows), - __expected__orders AS ( - SELECT order_id, status FROM expected_rows - ) - SELECT 1 - """.strip(), - expected_authored_cte_names=("expected_rows", "__source__raw_orders"), - expected_mock_model_names=(), - expected_mock_source_names=("raw_orders",), - expected_expected_model_names=("orders",), - ), - ], - ids=lambda case: case.description, -) -def test_given_model_sql_test_cte_variants_when_extracting_then_it_returns_expected_roles( - test_case: ExtractSqlTestCtesTestCase, -) -> None: - extracted_ctes: CompileSqlTestCtes = extract_test_sql( - sql=test_case.sql, file_label="tests/unit/orders.sql", mode=test_case.mode - ) - - assert isinstance(extracted_ctes.payload, CompileModelSqlTestCtes) - assert ( - tuple(cte.name for cte in extracted_ctes.payload.authored_ctes) - == test_case.expected_authored_cte_names - ) - assert extracted_ctes.payload.mock_model_names == test_case.expected_mock_model_names - assert extracted_ctes.payload.mock_source_names == test_case.expected_mock_source_names - assert extracted_ctes.payload.mock_seed_names == test_case.expected_mock_seed_names - assert extracted_ctes.payload.mock_dbt_ref_names == test_case.expected_mock_dbt_ref_names - assert ( - extracted_ctes.payload.mock_table_function_names - == test_case.expected_mock_table_function_names - ) - assert extracted_ctes.payload.expected_model_names == test_case.expected_expected_model_names - assert extracted_ctes.payload.assertion_names == test_case.expected_assertion_names - assert extracted_ctes.payload.macro_mocks == test_case.expected_macro_mocks - - -@pytest.mark.parametrize( - "test_case", - [ - ExtractSqlTestCtesTestCase( - description="extracts unsuffixed macro actual expected ctes in macro mode", - sql=""" - WITH input_values AS (SELECT ' PAID ' AS raw_status), - __macro_actual__ AS ( - SELECT @normalize_status("raw_status") AS status FROM input_values - ), - __macro_expected__ AS (SELECT 'paid' AS status) - SELECT 1 - """.strip(), - mode=SqlTestMode.MACRO, - expected_authored_cte_names=("input_values",), - expected_mock_model_names=(), - expected_mock_source_names=(), - expected_expected_model_names=(), - expected_macro_actual_cte_name="__macro_actual__", - expected_macro_expected_cte_name="__macro_expected__", - ), - ExtractSqlTestCtesTestCase( - description="extracts unsuffixed udf actual expected ctes in udf mode", - sql=""" - WITH input_values AS (SELECT 1250 AS amount_cents), - __udf_actual__ AS ( - SELECT __udf("format_cents")(amount_cents) AS formatted FROM input_values - ), - __udf_expected__ AS (SELECT '$12.50' AS formatted) - SELECT 1 - """.strip(), - mode=SqlTestMode.UDF, - expected_authored_cte_names=("input_values",), - expected_mock_model_names=(), - expected_mock_source_names=(), - expected_expected_model_names=(), - expected_macro_actual_cte_name="__udf_actual__", - expected_macro_expected_cte_name="__udf_expected__", - ), - ExtractSqlTestCtesTestCase( - description="extracts unsuffixed table function actual expected ctes in table_fn mode", - sql=""" - WITH __table_fn_actual__ AS ( - SELECT customer_id, order_id FROM __table_fn("customer_orders")(42) - ), - __table_fn_expected__ AS (SELECT 42 AS customer_id, 1 AS order_id) - SELECT 1 - """.strip(), - mode=SqlTestMode.TABLE_FN, - expected_authored_cte_names=(), - expected_mock_model_names=(), - expected_mock_source_names=(), - expected_expected_model_names=(), - expected_macro_actual_cte_name="__table_fn_actual__", - expected_macro_expected_cte_name="__table_fn_expected__", - ), - ExtractSqlTestCtesTestCase( - description="extracts macro ctes when the trailing select is omitted", - sql=""" - WITH __macro_actual__ AS (SELECT @double(2) AS value), - __macro_expected__ AS (SELECT 4 AS value) - """.strip(), - mode=SqlTestMode.MACRO, - expected_authored_cte_names=(), - expected_mock_model_names=(), - expected_mock_source_names=(), - expected_expected_model_names=(), - expected_macro_actual_cte_name="__macro_actual__", - expected_macro_expected_cte_name="__macro_expected__", - ), - ], - ids=lambda case: case.description, -) -def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_returns_expected_roles( - test_case: ExtractSqlTestCtesTestCase, -) -> None: - extracted_ctes: CompileSqlTestCtes = extract_test_sql( - sql=test_case.sql, file_label="tests/unit/orders.sql", mode=test_case.mode - ) - - assert isinstance(extracted_ctes.payload, CompileDirectLogicSqlTestCtes) - assert ( - tuple(cte.name for cte in extracted_ctes.payload.helper_ctes) - == test_case.expected_authored_cte_names - ) - assert extracted_ctes.payload.actual_cte.name == test_case.expected_macro_actual_cte_name - assert extracted_ctes.payload.expected_cte.name == test_case.expected_macro_expected_cte_name - - -@pytest.mark.parametrize( - "test_case", - [ - ExtractSqlTestCtesErrorTestCase( - description="raises when ceremonial select is wrapped in a final result cte", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id), - test_result AS (SELECT 1) - SELECT * FROM test_result - """.strip(), - expected_error_fragment="must end after its CTEs", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when assertion depends directly on expected result", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id), - __assert__expected_is_nonempty AS (SELECT * FROM __expected__orders) - SELECT 1 - """.strip(), - expected_error_fragment=( - "'__assert__expected_is_nonempty' must not depend on '__expected__orders'" - ), - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when unparsed assertion uses quoted expected result", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id), - __assert__expected_is_nonempty AS ( - SELECT @normalize_order(order_id) FROM "__expected__orders" - ) - SELECT 1 - """.strip(), - expected_error_fragment=( - "'__assert__expected_is_nonempty' must not depend on '__expected__orders'" - ), - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when assertion defines a shadowing expected result", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id), - __assert__empty AS ( - WITH __expected__orders AS (SELECT 2 AS order_id) - SELECT * FROM __expected__orders WHERE order_id < 0 - ) - SELECT 1 - """.strip(), - expected_error_fragment=( - "'__assert__empty' must not define expected result CTE '__expected__orders'" - ), - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when lone assertion defines a nested expected result", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __assert__empty AS ( - WITH __expected__orders AS (SELECT 2 AS order_id) - SELECT order_id FROM __expected__orders WHERE order_id < 0 - ) - SELECT 1 - """.strip(), - expected_error_fragment=( - "'__assert__empty' must not define expected result CTE '__expected__orders'" - ), - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when expected result depends directly on assertion", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __assert__positive_order_id AS (SELECT 1 AS order_id WHERE 1 < 1), - __expected__orders AS (SELECT order_id FROM __assert__positive_order_id) - SELECT 1 - """.strip(), - expected_error_fragment=( - "'__expected__orders' must not depend on '__assert__positive_order_id'" - ), - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when assertion depends indirectly on expected result", - sql=""" - WITH - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id), - expected_helper AS (SELECT * FROM __expected__orders), - __assert__expected_is_nonempty AS (SELECT * FROM expected_helper) - SELECT 1 - """.strip(), - expected_error_fragment=( - "'__assert__expected_is_nonempty' must not depend on " - "'__expected__orders' through 'expected_helper'" - ), - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when macro mock returns multiple columns", - sql=""" - WITH __macro__country AS (SELECT 'US', 'CA'), - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment=( - "macro mock '__macro__country' must be a single SELECT string literal" - ), - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when macro mock returns non string literal", - sql=""" - WITH __macro__country AS (SELECT 123), - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment=( - "macro mock '__macro__country' must be a single SELECT string literal" - ), - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when macro mock reads from a table", - sql=""" - WITH __macro__country AS (SELECT 'US' FROM values_table), - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment="with no FROM, UNION, or additional columns", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when macro mock uses union", - sql=""" - WITH __macro__country AS (SELECT 'US' UNION ALL SELECT 'CA'), - __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment="with no FROM, UNION, or additional columns", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when test sql does not start with top level with", - sql="SELECT 1", - expected_error_fragment="must declare mock CTEs and one __expected__", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when assertion cte omits assertion name", - sql=""" - WITH __source__raw_orders AS (SELECT 1), __assert__ AS (SELECT 1) - SELECT 1 - """.strip(), - expected_error_fragment="must use __assert__", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when cte body is unclosed", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id, - __expected__orders AS (SELECT 1 AS order_id) - SELECT 1 - """.strip(), - expected_error_fragment="contains an unclosed parenthesis", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when final select is not ceremonial select one", - sql=""" - WITH __source__raw_orders AS (SELECT 1), __expected__orders AS (SELECT 1 AS order_id) - SELECT 1 FROM __expected__orders - """.strip(), - expected_error_fragment="must end after its CTEs", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when final result cte is not ceremonial select one", - sql=""" - WITH - __source__raw_orders AS (SELECT 1), - __expected__orders AS (SELECT 1), - test_result AS (SELECT 2) - SELECT * FROM test_result - """.strip(), - expected_error_fragment="must end after its CTEs", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when fallback terminal read follows a set operation", - sql=""" - WITH - "__source__raw_orders" AS (SELECT 1), - "__expected__orders" AS (SELECT 1), - test_result AS (SELECT 1) - SELECT 2 UNION ALL SELECT * FROM test_result - """.strip(), - expected_error_fragment='CTE name "__source__raw_orders" must be an unquoted identifier', - ), - ExtractSqlTestCtesErrorTestCase( - description="raises for quoted CTE names instead of re-parsing the test", - sql=""" - WITH - "__source__raw_orders" AS MATERIALIZED (SELECT 1 AS order_id), - "__expected__orders" AS (SELECT order_id FROM "__source__raw_orders") - SELECT 1 - """.strip(), - expected_error_fragment='CTE name "__source__raw_orders" must be an unquoted identifier', - ), - ExtractSqlTestCtesErrorTestCase( - description="raises for materialized CTEs", - sql=""" - WITH __source__raw_orders AS MATERIALIZED (SELECT 1 AS order_id), - __expected__orders AS (SELECT order_id FROM __source__raw_orders) - SELECT 1 - """.strip(), - expected_error_fragment=( - "CTE '__source__raw_orders' must not use AS MATERIALIZED; materialization hints " - "are not supported in SQL test CTEs" - ), - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when expected cte omits target name", - sql=""" - WITH __source__raw_orders AS (SELECT 1), __expected__ AS (SELECT 1) - SELECT 1 - """.strip(), - expected_error_fragment="must use __expected__", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when expected cte uses select star", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS (SELECT * FROM __source__raw_orders) - SELECT 1 - """.strip(), - expected_error_fragment=r"must not use SELECT \* in __expected__orders CTEs", - ), - ExtractSqlTestCtesErrorTestCase( - description="rejects empty fixture marker for direct logic expected output", - sql=""" - WITH __macro_actual__ AS (SELECT 'paid' AS status), - __macro_expected__ AS (SELECT * FROM __empty_fixture()) - SELECT 1 - """.strip(), - mode=SqlTestMode.MACRO, - expected_error_fragment=r"must not use SELECT \* in __macro_expected__ CTEs", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when union branch projection order differs", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS ( - SELECT 1 AS order_id, 'paid' AS status - UNION ALL - SELECT 'created' AS status, 2 AS order_id - ) - SELECT 1 - """.strip(), - expected_error_fragment="projection names and order in every set-operation branch", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when union branch projection alias is missing", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS ( - SELECT CAST(1 AS INTEGER) AS order_id, CAST('paid' AS VARCHAR) AS status - UNION ALL - SELECT CAST(2 AS INTEGER), CAST('created' AS VARCHAR) AS status - ) - SELECT 1 - """.strip(), - expected_error_fragment="must alias every non-trivial __expected__orders projection", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when non union projection alias is missing", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS ( - SELECT CAST(1 AS INTEGER), CAST('paid' AS VARCHAR) AS status - ) - SELECT 1 - """.strip(), - expected_error_fragment="must alias every non-trivial __expected__orders projection", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when expected branch is not a select query", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id), - __expected__orders AS ( - SELECT 1 AS order_id - UNION ALL - VALUES (2) - ) - SELECT 1 - """.strip(), - expected_error_fragment="set-operation branch as a SELECT query", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when model mode includes macro actual cte", - sql=""" - WITH __macro_actual__ AS (SELECT @normalize_status('paid') AS status), - __macro_expected__ AS (SELECT 'paid' AS status) - SELECT 1 - """.strip(), - expected_error_fragment=r"use TEST \(mode macro\)", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when model mode includes udf actual cte", - sql=""" - WITH __udf_actual__ AS (SELECT __udf("format_cents")(1250) AS formatted), - __udf_expected__ AS (SELECT '$12.50' AS formatted) - SELECT 1 - """.strip(), - expected_error_fragment=r"use TEST \(mode udf\)", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when model mode includes table function actual cte", - sql=""" - WITH __table_fn_actual__ AS (SELECT * FROM __table_fn("customer_orders")(42)), - __table_fn_expected__ AS (SELECT 42 AS customer_id) - SELECT 1 - """.strip(), - expected_error_fragment=r"use TEST \(mode table_fn\)", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when macro mode includes model test cte", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id), - __macro_actual__ AS (SELECT @normalize_status('paid') AS status), - __macro_expected__ AS (SELECT 'paid' AS status) - SELECT 1 - """.strip(), - mode=SqlTestMode.MACRO, - expected_error_fragment="mode 'macro' but defines model-test CTE", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when macro expected calls macro", - sql=""" - WITH __macro_actual__ AS (SELECT @normalize_status('paid') AS status), - __macro_expected__ AS (SELECT @normalize_status('paid') AS status) - SELECT 1 - """.strip(), - mode=SqlTestMode.MACRO, - expected_error_fragment="__macro_expected__ must not call macros", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when macro helper calls macro", - sql=""" - WITH helper AS (SELECT @normalize_status('paid') AS status), - __macro_actual__ AS (SELECT status FROM helper), - __macro_expected__ AS (SELECT 'paid' AS status) - SELECT 1 - """.strip(), - mode=SqlTestMode.MACRO, - expected_error_fragment="helper CTE 'helper' must not call macros", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when udf mode includes model test cte", - sql=""" - WITH __source__raw_orders AS (SELECT 1 AS order_id), - __udf_actual__ AS (SELECT __udf("format_cents")(1250) AS formatted), - __udf_expected__ AS (SELECT '$12.50' AS formatted) - SELECT 1 - """.strip(), - mode=SqlTestMode.UDF, - expected_error_fragment="mode 'udf' but defines model-test CTE", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when udf expected calls udf", - sql=""" - WITH __udf_actual__ AS (SELECT __udf("format_cents")(1250) AS formatted), - __udf_expected__ AS (SELECT __udf("format_cents")(1250) AS formatted) - SELECT 1 - """.strip(), - mode=SqlTestMode.UDF, - expected_error_fragment="__udf_expected__.*must not call udf", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when udf helper calls udf", - sql=""" - WITH helper AS (SELECT __udf("format_cents")(1250) AS formatted), - __udf_actual__ AS (SELECT formatted FROM helper), - __udf_expected__ AS (SELECT '$12.50' AS formatted) - SELECT 1 - """.strip(), - mode=SqlTestMode.UDF, - expected_error_fragment="helper CTE 'helper'.*must not call udf", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when table function expected calls table function", - sql=""" - WITH __table_fn_actual__ AS (SELECT * FROM __table_fn("customer_orders")(42)), - __table_fn_expected__ AS ( - SELECT customer_id FROM __table_fn("customer_orders")(42) - ) - SELECT 1 - """.strip(), - mode=SqlTestMode.TABLE_FN, - expected_error_fragment="__table_fn_expected__.*must not call table_fn", - ), - ExtractSqlTestCtesErrorTestCase( - description="raises when table function helper calls table function", - sql=""" - WITH helper AS (SELECT * FROM __table_fn("customer_orders")(42)), - __table_fn_actual__ AS (SELECT * FROM helper), - __table_fn_expected__ AS (SELECT 42 AS customer_id) - SELECT 1 - """.strip(), - mode=SqlTestMode.TABLE_FN, - expected_error_fragment="helper CTE 'helper'.*must not call table_fn", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_sql_test_cte_variants_when_extracting_then_it_raises_clear_errors( - test_case: ExtractSqlTestCtesErrorTestCase, -) -> None: - with pytest.raises(ValueError, match=test_case.expected_error_fragment): - extract_test_sql(sql=test_case.sql, file_label="tests/unit/orders.sql", mode=test_case.mode) - - -@pytest.mark.parametrize( - "test_case", - ( - CteScannerMessageTestCase( - description="sql test without a with clause keeps sql test wording", - sql="SELECT * FROM orders", - expected_message=( - "SQL test 'tests/unit/orders.sql' must declare mock CTEs and one " - "__expected__ CTE in a top-level WITH clause" - ), - ), - CteScannerMessageTestCase( - description="sql test with an invalid cte name keeps sql test wording", - sql="WITH 1orders AS (SELECT 1) SELECT 1", - expected_message=( - "SQL test 'tests/unit/orders.sql' CTE name '1orders' must be an unquoted " - "identifier of ASCII letters, digits and underscores" - ), - ), - CteScannerMessageTestCase( - description="sql test without a ceremonial select keeps sql test wording", - sql=( - "WITH __source__raw_orders AS (SELECT 1 AS order_id), " - "__expected__orders AS (SELECT 1 AS order_id), " - "result AS (SELECT 1) SELECT * FROM result" - ), - expected_message=( - "SQL test 'tests/unit/orders.sql' must end after its CTEs; only an optional " - "ceremonial top-level `SELECT 1` may follow them" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_malformed_sql_test_ctes_when_extracting_then_messages_name_the_sql_test( - test_case: CteScannerMessageTestCase, -) -> None: - with pytest.raises(CompileInputError) as error_info: - _ = extract_test_sql(sql=test_case.sql, file_label="tests/unit/orders.sql") - - assert str(error_info.value) == test_case.expected_message - - -@pytest.mark.parametrize( - "test_case", - ( - CteScannerMessageTestCase( - description="expected model scan without AS keeps sql test wording", - sql="WITH __source__raw_orders (SELECT 1 AS order_id) SELECT 1", - expected_message="SQL test 'tests/unit/orders.sql' expected keyword AS", - ), - CteScannerMessageTestCase( - description="expected model scan without a target keeps sql test wording", - sql=( - "WITH __source__raw_orders AS (SELECT 1 AS order_id), " - "__expected__ AS (SELECT 1 AS order_id) SELECT 1" - ), - expected_message=( - "SQL test 'tests/unit/orders.sql' must use __expected__ to identify a target" - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_malformed_sql_test_ctes_when_scanning_expected_models_then_names_the_sql_test( - test_case: CteScannerMessageTestCase, -) -> None: - with pytest.raises(CompileInputError) as error_info: - _ = extract_sql_test_expected_model_names( - sql=test_case.sql, file_label="tests/unit/orders.sql", syntax=_GENERIC_SQL_SYNTAX - ) - - assert str(error_info.value) == test_case.expected_message - - -@pytest.mark.parametrize( - "test_case", - ( - CompleteCeremonialSelectTestCase( - description="adds the select after the last cte", - sql="WITH __source__orders AS (SELECT 1 AS id),\n__expected__orders AS (SELECT 1 AS id)", - expected_sql=( - "WITH __source__orders AS (SELECT 1 AS id),\n" - "__expected__orders AS (SELECT 1 AS id)\nSELECT 1" - ), - ), - CompleteCeremonialSelectTestCase( - description="adds the select before a trailing semicolon and comment", - sql="WITH __source__orders AS (SELECT 1 AS id);\n-- done", - expected_sql="WITH __source__orders AS (SELECT 1 AS id)\nSELECT 1;\n-- done", - ), - CompleteCeremonialSelectTestCase( - description="ignores parentheses in a trailing comment and string", - sql="WITH __source__orders AS (SELECT ')' AS id) -- closes (\n", - expected_sql="WITH __source__orders AS (SELECT ')' AS id)\nSELECT 1 -- closes (\n", - ), - CompleteCeremonialSelectTestCase( - description="keeps an authored ceremonial select unchanged", - sql="WITH __source__orders AS (SELECT 1 AS id)\nSELECT 1", - expected_sql="WITH __source__orders AS (SELECT 1 AS id)\nSELECT 1", - ), - CompleteCeremonialSelectTestCase( - description="keeps another final statement for the scanner to reject", - sql="WITH __source__orders AS (SELECT 1 AS id)\nSELECT * FROM __source__orders", - expected_sql="WITH __source__orders AS (SELECT 1 AS id)\nSELECT * FROM __source__orders", - ), - CompleteCeremonialSelectTestCase( - description="keeps sql without a with clause unchanged", - sql="SELECT 1", - expected_sql="SELECT 1", - ), - CompleteCeremonialSelectTestCase( - description="keeps an unclosed cte unchanged", - sql="WITH __source__orders AS (SELECT 1 AS id", - expected_sql="WITH __source__orders AS (SELECT 1 AS id", - ), - ), - ids=lambda case: case.description, -) -def test_given_test_sql_when_completing_ceremonial_select_then_only_omitted_select_is_added( - test_case: CompleteCeremonialSelectTestCase, -) -> None: - completed: str = complete_omitted_ceremonial_select( - sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX - ) - - assert completed == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py index 72cf455472..c828ed11da 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_typed_macro_references.py @@ -7,7 +7,8 @@ import pytest from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( MacroGeneratedReferenceCollectionTestCase, MacroGeneratedReferenceErrorTestCase, @@ -396,7 +397,7 @@ def test_given_typed_macro_arguments_when_compiling_then_every_argument_is_a_dep }, ) - project: CompiledProject = compile_and_assemble(project_dir=tmp_path) + project: CompiledProjectView = compile_and_assemble(project_dir=tmp_path) - summary: CompiledModel = {model.name: model for model in project.models}["order_summary"] + summary: CompiledModelView = {model.name: model for model in project.models}["order_summary"] assert frozenset(dep.name for dep in summary.deps) == test_case.expected_dependency_names diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py index f203b5f056..c33c1c63cb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_cursor_start.py @@ -6,7 +6,7 @@ import pytest from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( @@ -48,7 +48,7 @@ def test_given_cursor_start_layers_when_building_compile_inputs_then_model_uses_ write_repo_files(tmp_path, test_case.repo_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) @@ -97,7 +97,7 @@ def test_given_cursor_safety_layers_when_compiling_then_model_fields_use_model_p write_repo_files(tmp_path, test_case.repo_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_main.py b/tests/unit/src/sqlbuild/compiler/compile/test_main.py index 3ed278a55d..0e6cc220e6 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_main.py @@ -10,9 +10,8 @@ from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompileModelConfig, - CompileProjectInputs, - CompileSqlFunctionInput, ) +from sqlbuild.compiler.compile.types import CompileInputsView, FunctionInput from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, @@ -2767,7 +2766,7 @@ def test_given_discovered_inputs_when_building_compile_inputs_then_it_attaches_m write_repo_files(tmp_path, test_case.repo_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, selected_target=test_case.selected_target, @@ -3125,7 +3124,7 @@ def main(): ) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=replace( DUCKDB_COMPILE_ADAPTER_CONTEXT, @@ -3133,7 +3132,7 @@ def main(): ), ) - functions: dict[str, CompileSqlFunctionInput] = { + functions: dict[str, FunctionInput] = { entry.name: entry for entry in compile_inputs.sql_function_inputs } assert functions["defaulted"].tags == ("reference", "daily") @@ -3162,7 +3161,7 @@ def main(): functions["unqualified"].fingerprint_schema, ) == test_case.expected_unqualified_fingerprint_namespace - physical_inputs: CompileProjectInputs = build_compile_inputs( + physical_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=replace( DUCKDB_COMPILE_ADAPTER_CONTEXT, @@ -3170,7 +3169,7 @@ def main(): ), selected_target="physical", ) - physical_functions: dict[str, CompileSqlFunctionInput] = { + physical_functions: dict[str, FunctionInput] = { entry.name: entry for entry in physical_inputs.sql_function_inputs } assert ( @@ -3300,7 +3299,7 @@ def test_given_rejected_reference_calls_when_building_compile_inputs_then_report ) -> None: write_repo_files(tmp_path, test_case.repo_files) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) @@ -4746,7 +4745,7 @@ def test_given_authored_retention_header_when_building_inputs_then_typed_policy_ write_repo_files(tmp_path, test_case.repo_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) @@ -4846,7 +4845,7 @@ def test_given_model_referencing_seed_when_building_compile_inputs_then_succeeds write_repo_files(tmp_path, test_case.repo_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py index d162c1b057..1117887f2f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py +++ b/tests/unit/src/sqlbuild/compiler/compile/test_resolved_connection.py @@ -6,7 +6,7 @@ import pytest from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( @@ -56,7 +56,7 @@ def test_given_resolved_connection_when_building_compile_inputs_then_uses_expect write_repo_files(tmp_path, test_case.repo_files) discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered_inputs, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, resolved_connection=test_case.resolved_connection, diff --git a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/_test_types.py index 8efe294dff..a1e69a1aff 100644 --- a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/_test_types.py @@ -1,14 +1,6 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class DeclaredTypeSpanTestCase: - description: str - contents: str - declared_type: str - expected_value: str - - @dataclass(frozen=True) class DynamicContractAdoptionTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py index 7738441e93..fb8d66513d 100644 --- a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py +++ b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py @@ -2,57 +2,19 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledModel -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.contract_adoption._helpers.compare import compare_evidence from sqlbuild.compiler.contract_adoption._helpers.repository_edits import ( - _declared_type_span, edit_model, ) from sqlbuild.compiler.contract_adoption.models import ContractEvidence -from sqlbuild.compiler.discovery.models import ModelHeaderColumnSpan from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily from tests.unit.src.sqlbuild.compiler.contract_adoption._helpers._test_types import ( - DeclaredTypeSpanTestCase, DynamicContractAdoptionTestCase, ) from tests.unit.src.sqlbuild.compiler.contracts.helpers import make_contract_project -@pytest.mark.parametrize( - "test_case", - [ - DeclaredTypeSpanTestCase( - description="nested type argument precedes the declared column type", - contents="amount (audits [custom(type INT)], type INT)", - declared_type="INT", - expected_value="INT", - ) - ], - ids=lambda case: case.description, -) -def test_given_nested_type_key_when_locating_declared_type_then_returns_top_level_value( - test_case: DeclaredTypeSpanTestCase, -) -> None: - contents: str = test_case.contents - metadata_start: int = contents.index("audits") - metadata_end: int = contents.rindex(")") - expected_start: int = contents.rindex(test_case.expected_value) - - result: tuple[int, int] | None = _declared_type_span( - contents=contents, - column_span=ModelHeaderColumnSpan( - entry_start=0, - entry_end=len(contents), - metadata_start=metadata_start, - metadata_end=metadata_end, - ), - declared_type=test_case.declared_type, - ) - - assert result == (expected_start, expected_start + len(test_case.expected_value)) - - @pytest.mark.parametrize( "test_case", (DynamicContractAdoptionTestCase(description="preserve dynamic family generation"),), @@ -75,7 +37,7 @@ def test_given_dynamic_family_when_generating_contract_then_does_not_freeze_runt ); SELECT * FROM category_amounts """ - model: CompiledModel = make_contract_project( + model: CompiledModelView = make_contract_project( declared_columns=(("customer_id", "INTEGER"),), inferred_columns=(("customer_id", "INTEGER"),), type_enforcement=True, diff --git a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py index 1fb7cb04b1..3df66a97ac 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py @@ -12,7 +12,7 @@ DynamicColumnContractProof, InferredColumn, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import ( SchemaAuditInstance, @@ -41,7 +41,7 @@ def make_contract_project( authored_sql: str = "", dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = (), dynamic_column_contract: DynamicColumnContractProof | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """Build a compiled project for contract validation tests.""" key: CompiledObjectKey = CompiledObjectKey( @@ -122,7 +122,7 @@ def make_promotion_project( *, model_configs: tuple[tuple[str, dict[str, object]], ...], table_promotion_mode: str | None, -) -> CompiledProject: +) -> CompiledProjectView: """Build a compiled project of models with the given configs and promotion setting.""" return CompiledProject( diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py index 5e7458b8de..bcad829375 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py @@ -19,6 +19,7 @@ ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, + CompiledProjectView, CompiledResourceType, FunctionLanguage, ) @@ -41,6 +42,7 @@ SourceColumnEntry, SourceEntry, ) +from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes def build_dag_artifact_test_graph() -> ProjectGraph: @@ -98,7 +100,7 @@ def raw_orders_loader() -> list[dict[str, object]]: qualified_name="analytics_dev.normalize_email", logical_schema="analytics", ) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="run-1", effective_target_name="dev", effective_connection={}, @@ -242,7 +244,7 @@ def raw_orders_loader() -> list[dict[str, object]]: function_key: (), model_key: (source_key, seed_key, function_key), } - return ProjectGraph( + return project_graph_from_indexes( project=project, upstream_deps=upstream_deps, downstream_deps={ diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 489961c6dd..5226a56150 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -12,17 +12,15 @@ from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, - CompileProjectInputs, ) +from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.dag.main.build import build_dag_json from sqlbuild.compiler.dag.types import NodeKind from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner._helpers.graph.selector_indexes import ( - build_model_tag_index_impl, -) from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) @@ -35,6 +33,7 @@ DagResourceNamespaceTestCase, ) from tests.unit.src.sqlbuild.compiler.dag.main.helpers import build_dag_artifact_test_graph +from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes @pytest.mark.parametrize( @@ -116,7 +115,9 @@ def test_given_project_graph_when_building_dag_artifact_then_includes_assets_edg assert nodes_by_id["loader:shared_order_feed"]["kind"] == "loader" assert tuple(checks[0]["checked_asset_ids"]) == ("model:orders",) assert checks[0]["severity"] == "warn" - tag_index: dict[str, frozenset[CompiledObjectKey]] = build_model_tag_index_impl(graph.project) + tag_index: dict[str, frozenset[CompiledObjectKey]] = project_lineage_views( + graph.project + ).tag_index assert tag_index["reference"] == frozenset({graph.project.seeds[0].key}) assert tag_index["normalization"] == frozenset({graph.project.functions[0].key}) @@ -231,12 +232,12 @@ def test_given_resource_namespace_defaults_when_building_dag_then_targets_retain }, ) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - inputs: CompileProjectInputs = build_compile_inputs( + inputs: CompileInputsView = build_compile_inputs( discovered_inputs=discovered, adapter_context=DUCKDB_COMPILE_ADAPTER_CONTEXT, ) - project: CompiledProject = assemble_compiled_project(inputs=inputs) - graph: ProjectGraph = ProjectGraph( + project: CompiledProjectView = assemble_compiled_project(inputs=inputs) + graph: ProjectGraph = project_graph_from_indexes( project=project, upstream_deps={}, downstream_deps={}, @@ -317,8 +318,8 @@ def test_given_qualified_loader_destination_when_building_dag_then_parses_loader graph: ProjectGraph = build_dag_artifact_test_graph() graph = replace( graph, - project=replace( - graph.project, + project=project_with( + project=graph.project, effective_target_database="default_db", effective_target_schema="default_schema", loader_functions=( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py index a1639b0983..651cd18e11 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_helpers.py @@ -25,14 +25,24 @@ from sqlbuild.compiler.discovery._helpers.native.model_files import discover_native_model_files from sqlbuild.compiler.discovery._helpers.native.sql_test_files import discover_native_test_files from sqlbuild.compiler.discovery._helpers.native.yaml_files import discover_native_schema_files -from sqlbuild.compiler.discovery._helpers.sql.model_files import match_model_header from sqlbuild.compiler.discovery._helpers.yml.project import ( load_local_config, load_project_config, ) from sqlbuild.compiler.discovery.classes.directory_snapshot import DirectorySnapshot -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, ModelHeaderMatch - +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + +# Test-only reference grammar of the MODEL header the native matcher replaced. +_REFERENCE_MODEL_HEADER_PATTERN: re.Pattern[str] = re.compile( + r"^\s*MODEL\s*\(" + r"(?P
(?>(?:[^\"')\\]|\\.)++" + r"|\"(?:[^\"\\]|\\.)*+\"" + r"|'(?:[^'\\]|\\.)*+'" + r"|[\"'](?:[^\\)]|\\.)*+" + r"|\)(?!\s*;))*+)" + r"\)\s*;\s*(?P.*)\Z", + re.DOTALL, +) _DIRECTORY_NAMES: tuple[str, ...] = ( "orders", "customers", @@ -202,16 +212,16 @@ def header_match_mismatches(*, seed: int, count: int) -> tuple[str, ...]: def _python_header_offsets(contents: str) -> tuple[int, int, int] | None: - return _mapped_offsets(match_model_header(contents)) + return _mapped_offsets(_REFERENCE_MODEL_HEADER_PATTERN.match(contents)) -def _mapped_offsets(header_match: ModelHeaderMatch | None) -> tuple[int, int, int] | None: +def _mapped_offsets(header_match: re.Match[str] | None) -> tuple[int, int, int] | None: return { True: lambda: None, False: lambda: ( - cast(ModelHeaderMatch, header_match).header_start, - cast(ModelHeaderMatch, header_match).header_end, - cast(ModelHeaderMatch, header_match).sql_start, + cast(re.Match[str], header_match).start("header"), + cast(re.Match[str], header_match).end("header"), + cast(re.Match[str], header_match).start("sql"), ), }[header_match is None]() diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 249a562b0a..5c38e9fbdd 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -1038,3 +1038,21 @@ class TargetDiffConfigErrorTestCase: project_contents: str expected_error_fragment: str local_contents: str = "" + + +@dataclass(frozen=True) +class DeepProjectConnectionTestCase: + """Test case for project TOML nested deeper than the native parser's own limit.""" + + description: str + depth: int + expected_depth: int + + +@dataclass(frozen=True) +class OversizedDurationTestCase: + """Test case for a duration with more digits than Python's int() converts.""" + + description: str + section: str + expected_error_fragment: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py index 510e54f6db..9dda0fe2b5 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_declarations.py @@ -6,9 +6,7 @@ import pytest -from sqlbuild.compiler.discovery._helpers.sql.declarations import ( - parse_model_constant_declarations, -) +import sqlbuild._native as _native from sqlbuild.compiler.discovery.exceptions import DeclarationParseError from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, @@ -196,10 +194,11 @@ def test_given_typed_public_constants_when_parsing_then_returns_normalized_value def test_given_local_shorthand_and_wrapper_constants_when_parsing_then_honors_options( test_case: ParseLocalTypedConstantsTestCase, ) -> None: - declarations: tuple[ConstantDeclaration, ...] = parse_model_constant_declarations( - raw_value=test_case.raw_value, - model_name="orders", - relative_path=Path("models/orders.sql"), + declarations: tuple[ConstantDeclaration, ...] = cast( + tuple[ConstantDeclaration, ...], + _native.model_header_declarations( + (None, test_case.raw_value), "orders", Path("models/orders.sql") + )[1], ) assert ( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index 9e1cd99b0d..8b05297382 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from decimal import Decimal from pathlib import Path @@ -23,6 +24,7 @@ from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ColumnContractModeConfigErrorTestCase, ColumnContractModeConfigTestCase, + DeepProjectConnectionTestCase, DiffConfigTestCase, EffectiveBatchDefaultTestCase, FutureCursorConfigErrorTestCase, @@ -44,6 +46,7 @@ MicrobatchLimitConfigErrorTestCase, MicrobatchLimitConfigTestCase, MissingMigrationOriginConfigTestCase, + OversizedDurationTestCase, StartCursorConfigTestCase, TargetExecutionLimitsConfigTestCase, ) @@ -2077,3 +2080,61 @@ def test_given_invalid_column_contract_mode_when_loading_project_then_error_is_r with pytest.raises(ProjectConfigError, match=test_case.expected_error_fragment): load_project_config(project_dir=tmp_path) + + +@pytest.mark.parametrize( + "test_case", + [ + DeepProjectConnectionTestCase( + description="arrays nested three hundred deep", depth=300, expected_depth=300 + ), + ], + ids=lambda case: case.description, +) +def test_given_deeply_nested_connection_when_loading_project_then_tomllib_nesting_is_kept( + test_case: DeepProjectConnectionTestCase, + tmp_path: Path, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "demo"\nadapter = "duckdb"\n[connection]\n' + f"nested = {'[' * test_case.depth}1{']' * test_case.depth}\n", + encoding="utf-8", + ) + + config: ProjectConfig = load_project_config(project_dir=tmp_path) + + nested: object = config.connection["nested"] + depth: int = 0 + while isinstance(nested, list): + nested = nested[0] + depth += 1 + assert depth == test_case.expected_depth + + +@pytest.mark.parametrize( + "test_case", + [ + OversizedDurationTestCase( + description="diff artifact TTL", + section='[diff]\nquery_artifact_ttl = "{digits}h"\n', + expected_error_fragment="Exceeds the limit (4300 digits)", + ), + OversizedDurationTestCase( + description="lifecycle shutdown timeout", + section='[sinks.lifecycle]\nshutdown_timeout = "{digits}s"\n', + expected_error_fragment="value has 4301 digits", + ), + ], + ids=lambda case: case.description, +) +def test_given_duration_beyond_int_digit_limit_when_loading_project_then_value_error_is_raised( + test_case: OversizedDurationTestCase, + tmp_path: Path, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "demo"\nadapter = "duckdb"\n' + test_case.section.format(digits="1" * 4301), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match=re.escape(test_case.expected_error_fragment)): + _ = load_project_config(project_dir=tmp_path) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py index ff09c62576..e6a57f73e6 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/helpers.py @@ -11,7 +11,6 @@ from sqlbuild.compiler.discovery._helpers.native.sql_test_files import ( parse_native_sql_test_contents, ) -from sqlbuild.compiler.discovery._helpers.sql.model_files import project_native_header_values from sqlbuild.compiler.discovery._helpers.sql.scenarios import build_sql_scenario_file from sqlbuild.compiler.discovery._helpers.yml.schema import parse_loaded_schema_yml from sqlbuild.compiler.discovery._helpers.yml.sources import parse_loaded_sources_yml @@ -63,7 +62,7 @@ def parse_model_sql(*, contents: str, file_path: Path) -> tuple[dict[str, object """Parse model contents as model discovery does, returning header values and query SQL.""" payload: tuple[object, ...] = _parsed_model(contents=contents, file_path=file_path) - return project_native_header_values(cast(dict[str, object], payload[2])), str(payload[5]) + return cast(dict[str, object], payload[2]), str(payload[5]) def model_header_column_locations( @@ -115,7 +114,7 @@ def parse_sql_scenario_file( ) _raise_payload_error(payload=payload, file_path=file_path) return build_sql_scenario_file( - header_values=project_native_header_values(cast(dict[str, object], payload[2])), + header_values=cast(dict[str, object], payload[2]), sql_body=str(payload[3]), contents=str(payload[1]), file_path=file_path, diff --git a/tests/unit/src/sqlbuild/compiler/graph/helpers.py b/tests/unit/src/sqlbuild/compiler/graph/helpers.py new file mode 100644 index 0000000000..1adab64345 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/graph/helpers.py @@ -0,0 +1,33 @@ +"""Shared builders for tests that hand-assemble a project graph.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping + +from sqlbuild.compiler.compile.models import CompiledObjectKey +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.graph.main._native_graph_from_views import native_graph_from_views +from sqlbuild.compiler.pipeline.models import ProjectGraph + + +def project_graph_from_indexes( + *, + project: CompiledProjectView, + upstream_deps: Mapping[CompiledObjectKey, Iterable[CompiledObjectKey]], + downstream_deps: Mapping[CompiledObjectKey, Iterable[CompiledObjectKey]], + tag_index: Mapping[str, Iterable[CompiledObjectKey]], + path_index: Mapping[CompiledObjectKey, str], + all_keys: Mapping[str, CompiledObjectKey], +) -> ProjectGraph: + """A natively held graph over hand-built indexes, keeping their order.""" + + return ProjectGraph( + project=project, + native=native_graph_from_views( + all_keys=all_keys, + upstream=upstream_deps, + downstream=downstream_deps, + tag_index=tag_index, + path_index=path_index, + ), + ) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py index b1af5052b4..2206c7981f 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/_test_types.py @@ -38,7 +38,6 @@ class RichLineageAnalysisScalingTestCase: unrelated_model_counts: tuple[int, ...] expected_schema_builds: int expected_analysis_calls: int - expected_schema_tables_per_call: tuple[tuple[str, ...], ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py index 4e2a123983..14d2b573af 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py @@ -11,17 +11,22 @@ CompileModelConfig, InferredColumn, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompiledSeedView, +) from sqlbuild.compiler.discovery.models import DiscoveredSchemaFile, DiscoveredSeedFile from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SettingsConfig def make_compiled_project( *, - models: tuple[CompiledModel, ...], - seeds: tuple[CompiledSeed, ...] = (), + models: tuple[CompiledModelView, ...], + seeds: tuple[CompiledSeedView, ...] = (), sql_analysis_enabled: bool = True, -) -> CompiledProject: +) -> CompiledProjectView: return CompiledProject( run_id="test-run", effective_target_name=None, @@ -38,7 +43,7 @@ def make_compiled_model( name: str, query_sql: str, inferred_columns: tuple[str, ...] | None, -) -> CompiledModel: +) -> CompiledModelView: return CompiledModel( key=CompiledObjectKey(CompiledResourceType.MODEL, name), deps=(), @@ -59,7 +64,7 @@ def make_compiled_model( ) -def make_compiled_seed(*, name: str, columns: tuple[str, ...]) -> CompiledSeed: +def make_compiled_seed(*, name: str, columns: tuple[str, ...]) -> CompiledSeedView: return CompiledSeed( key=CompiledObjectKey(CompiledResourceType.SEED, name), deps=(), @@ -94,13 +99,13 @@ def edge_label(source_name: str, source_column: str, target_name: str, target_co def make_chain_models( *, chain_length: int, unrelated_model_count: int -) -> tuple[CompiledModel, ...]: - root: CompiledModel = make_compiled_model( +) -> tuple[CompiledModelView, ...]: + root: CompiledModelView = make_compiled_model( name="orders_0", query_sql="SELECT 1 AS order_id, 10 AS amount", inferred_columns=("order_id", "amount"), ) - chain: tuple[CompiledModel, ...] = tuple( + chain: tuple[CompiledModelView, ...] = tuple( make_compiled_model( name=f"orders_{index}", query_sql=f'SELECT order_id, amount + 1 AS amount FROM __ref("orders_{index - 1}")', @@ -108,7 +113,7 @@ def make_chain_models( ) for index in range(1, chain_length) ) - unrelated: tuple[CompiledModel, ...] = tuple( + unrelated: tuple[CompiledModelView, ...] = tuple( make_compiled_model( name=f"inventory_{index}", query_sql="SELECT 1 AS product_id, 5 AS quantity", diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 630cc235fe..6a2d2784f8 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -5,15 +5,18 @@ import pytest +import sqlbuild._native as native from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, - CompiledModel, - CompiledProject, ) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage._helpers import rich_columns -from sqlbuild.compiler.lineage._helpers.columns import _build_schema_mapping +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.lineage._helpers import native_rich_columns from sqlbuild.compiler.lineage.main.columns import build_project_column_lineage from sqlbuild.compiler.lineage.models import ColumnLineage, ModelColumnLineage, ProjectColumnLineage from sqlbuild.compiler.lineage.types import ( @@ -21,7 +24,6 @@ ColumnLineageMode, ColumnTransformKind, ) -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from tests.unit.src.sqlbuild.compiler.lineage._test_types import ( ColumnLineageAnalyzerTestCase, ExpectedCountTestCase, @@ -154,7 +156,7 @@ def test_given_compiled_project_when_building_column_lineage_then_infers_expected_column( test_case: ColumnLineageAnalyzerTestCase, ) -> None: - upstream_models: tuple[CompiledModel, ...] = tuple( + upstream_models: tuple[CompiledModelView, ...] = tuple( make_compiled_model( name=model_name, query_sql="SELECT 1", @@ -162,12 +164,12 @@ def test_given_compiled_project_when_building_column_lineage_then_infers_expecte ) for model_name, columns in test_case.upstream_model_columns.items() ) - target_model: CompiledModel = make_compiled_model( + target_model: CompiledModelView = make_compiled_model( name=test_case.model_name, query_sql=test_case.query_sql, inferred_columns=test_case.inferred_columns, ) - project: CompiledProject = make_compiled_project( + project: CompiledProjectView = make_compiled_project( models=upstream_models + (target_model,), seeds=tuple( make_compiled_seed(name=seed_name, columns=columns) @@ -286,7 +288,7 @@ def test_given_compiled_project_when_building_column_lineage_then_infers_expecte def test_given_compiled_project_when_building_fast_column_lineage_then_infers_expected_column( test_case: ColumnLineageAnalyzerTestCase, ) -> None: - upstream_models: tuple[CompiledModel, ...] = tuple( + upstream_models: tuple[CompiledModelView, ...] = tuple( make_compiled_model( name=model_name, query_sql="SELECT 1", @@ -294,12 +296,12 @@ def test_given_compiled_project_when_building_fast_column_lineage_then_infers_ex ) for model_name, columns in test_case.upstream_model_columns.items() ) - target_model: CompiledModel = make_compiled_model( + target_model: CompiledModelView = make_compiled_model( name=test_case.model_name, query_sql=test_case.query_sql, inferred_columns=test_case.inferred_columns, ) - project: CompiledProject = make_compiled_project( + project: CompiledProjectView = make_compiled_project( models=upstream_models + (target_model,), seeds=tuple( make_compiled_seed(name=seed_name, columns=columns) @@ -349,7 +351,7 @@ def test_given_compiled_project_when_building_fast_column_lineage_then_infers_ex def test_given_select_star_when_building_column_lineage_then_expands_known_schema_columns( test_case: ColumnLineageAnalyzerTestCase, ) -> None: - project: CompiledProject = make_compiled_project( + project: CompiledProjectView = make_compiled_project( models=( make_compiled_model( name="orders", @@ -391,7 +393,7 @@ def test_given_select_star_when_building_column_lineage_then_expands_known_schem def test_given_linear_project_when_listing_column_consumers_then_returns_expected_edges( test_case: ProjectLineageGraphTestCase, ) -> None: - project: CompiledProject = make_compiled_project( + project: CompiledProjectView = make_compiled_project( models=( make_compiled_model(name="a", query_sql="SELECT 1 AS id", inferred_columns=("id",)), make_compiled_model( @@ -426,16 +428,18 @@ def test_given_linear_project_when_listing_column_consumers_then_returns_expecte def test_given_compact_analysis_facts_when_building_fast_lineage_then_preserves_public_graph( test_case: ExpectedCountTestCase, ) -> None: - upstream: CompiledModel = make_compiled_model( + upstream: CompiledModelView = make_compiled_model( name="orders", query_sql="SELECT 1 AS order_id", inferred_columns=("order_id",), ) - target: CompiledModel = replace( - make_compiled_model( - name="order_summary", - query_sql='SELECT order_id FROM __ref("orders")', - inferred_columns=("order_id",), + target: CompiledModelView = replace( + compiled_model_record( + make_compiled_model( + name="order_summary", + query_sql='SELECT order_id FROM __ref("orders")', + inferred_columns=("order_id",), + ) ), fast_lineage_columns=( CompiledLineageColumnFact( @@ -532,7 +536,7 @@ def test_given_compact_analysis_facts_when_building_fast_lineage_then_preserves_ def test_given_cte_sourced_column_when_building_rich_column_lineage_then_resolves_upstream( test_case: ColumnLineageAnalyzerTestCase, ) -> None: - upstream_models: tuple[CompiledModel, ...] = tuple( + upstream_models: tuple[CompiledModelView, ...] = tuple( make_compiled_model( name=model_name, query_sql="SELECT 1", @@ -540,12 +544,12 @@ def test_given_cte_sourced_column_when_building_rich_column_lineage_then_resolve ) for model_name, columns in test_case.upstream_model_columns.items() ) - target_model: CompiledModel = make_compiled_model( + target_model: CompiledModelView = make_compiled_model( name=test_case.model_name, query_sql=test_case.query_sql, inferred_columns=test_case.inferred_columns, ) - project: CompiledProject = make_compiled_project( + project: CompiledProjectView = make_compiled_project( models=upstream_models + (target_model,), ) @@ -593,7 +597,7 @@ def test_given_cte_sourced_column_when_building_rich_column_lineage_then_resolve def test_given_postgres_dialect_when_building_rich_column_lineage_then_uses_polyglot_name( test_case: ColumnLineageAnalyzerTestCase, ) -> None: - upstream_models: tuple[CompiledModel, ...] = tuple( + upstream_models: tuple[CompiledModelView, ...] = tuple( make_compiled_model( name=model_name, query_sql="SELECT 1", @@ -601,12 +605,12 @@ def test_given_postgres_dialect_when_building_rich_column_lineage_then_uses_poly ) for model_name, columns in test_case.upstream_model_columns.items() ) - target_model: CompiledModel = make_compiled_model( + target_model: CompiledModelView = make_compiled_model( name=test_case.model_name, query_sql=test_case.query_sql, inferred_columns=test_case.inferred_columns, ) - project: CompiledProject = make_compiled_project(models=upstream_models + (target_model,)) + project: CompiledProjectView = make_compiled_project(models=upstream_models + (target_model,)) result: ProjectColumnLineage | None = build_project_column_lineage( project=project, @@ -645,7 +649,7 @@ def test_given_postgres_dialect_when_building_rich_column_lineage_then_uses_poly def test_given_sql_analysis_disabled_when_building_column_lineage_then_returns_no_lineage( test_case: SqlAnalysisDisabledLineageTestCase, ) -> None: - project: CompiledProject = make_compiled_project( + project: CompiledProjectView = make_compiled_project( models=( make_compiled_model( name="orders_out", @@ -670,12 +674,6 @@ def test_given_sql_analysis_disabled_when_building_column_lineage_then_returns_n unrelated_model_counts=(0, 250), expected_schema_builds=1, expected_analysis_calls=4, - expected_schema_tables_per_call=( - (), - ("__sqlbuild_model__orders_0",), - ("__sqlbuild_model__orders_1",), - ("__sqlbuild_model__orders_2",), - ), ) ], ids=lambda case: case.description, @@ -684,29 +682,29 @@ def test_given_unrelated_models_when_tracing_rich_lineage_then_analysis_work_sta test_case: RichLineageAnalysisScalingTestCase, monkeypatch: pytest.MonkeyPatch, ) -> None: - polyglot_module: Any = import_polyglot_sql() - analyze_query: Any = polyglot_module.analyze_query - schema_builds: list[CompiledProject] = [] - analysed_tables: list[tuple[str, ...]] = [] + build_lineage: Any = native.build_rich_column_lineage + schema_resources: Any = native_rich_columns._schema_resources + schema_builds: list[CompiledProjectView] = [] + analysed_queries: list[str] = [] - def counting_schema_mapping(project: CompiledProject) -> dict[str, dict[str, str]]: + def counting_schema_mapping(project: CompiledProjectView) -> Any: schema_builds.append(project) - return _build_schema_mapping(project) + return schema_resources(project) - def recording_analyze_query(sql: str, options: dict[str, Any]) -> object: - analysed_tables.append(tuple(table["name"] for table in options["schema"]["tables"])) - return analyze_query(sql, options) + def recording_native_lineage(request: Any) -> Any: + analysed_queries.extend(request[2]) + return build_lineage(request) - monkeypatch.setattr(rich_columns, "_build_schema_mapping", counting_schema_mapping) - monkeypatch.setattr(polyglot_module, "analyze_query", recording_analyze_query) + monkeypatch.setattr(native_rich_columns, "_schema_resources", counting_schema_mapping) + monkeypatch.setattr(native, "build_rich_column_lineage", recording_native_lineage) chain_names: frozenset[str] = frozenset( f"orders_{index}" for index in range(test_case.chain_length) ) edges_by_project_size: list[tuple[str, ...]] = [] for unrelated_model_count in test_case.unrelated_model_counts: schema_builds.clear() - analysed_tables.clear() - project: CompiledProject = make_compiled_project( + analysed_queries.clear() + project: CompiledProjectView = make_compiled_project( models=make_chain_models( chain_length=test_case.chain_length, unrelated_model_count=unrelated_model_count, @@ -719,8 +717,7 @@ def recording_analyze_query(sql: str, options: dict[str, Any]) -> object: assert result is not None assert len(schema_builds) == test_case.expected_schema_builds - assert len(analysed_tables) == test_case.expected_analysis_calls - assert tuple(analysed_tables) == test_case.expected_schema_tables_per_call + assert len(analysed_queries) == test_case.expected_analysis_calls edges_by_project_size.append( tuple( edge_label( diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py index d8d73b71f8..9d68bb05c8 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/_test_types.py @@ -5,7 +5,6 @@ from pathlib import Path from sqlbuild.compiler.compile.models import DeclarationResolutionContext, MacroContext -from sqlbuild.python_nodes.models import SqlResourceRef @dataclass(frozen=True) @@ -29,16 +28,6 @@ class ContextValueTestCase: expected_stored: bool -@dataclass(frozen=True) -class CallClassStoreTextTestCase: - """A call class's context and relations, compared with a plain class of the same macros.""" - - description: str - context_token: str | None - prior_relations: tuple[SqlResourceRef, ...] | None - expected_equal: bool - - @dataclass(frozen=True) class MacroStoreTokenTestCase: """A change to a loaded macro and whether its stored results stay valid.""" diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py index 41d079e5b6..42cca1fbde 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/_helpers/test_store_keys.py @@ -6,14 +6,10 @@ import pytest from sqlbuild.compiler.macro_bridge._helpers.store_keys import ( - call_class_store_text, context_store_token, macro_store_token, ) -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind from tests.unit.src.sqlbuild.compiler.macro_bridge._helpers._test_types import ( - CallClassStoreTextTestCase, ContextStoreTokenTestCase, ContextValueTestCase, MacroStoreTokenTestCase, @@ -175,46 +171,3 @@ def test_given_loaded_macro_change_when_naming_it_then_only_the_same_file_and_so ) assert (original == changed) is test_case.expected_equal - - -@pytest.mark.parametrize( - "test_case", - [ - CallClassStoreTextTestCase( - description="same class", context_token=None, prior_relations=None, expected_equal=True - ), - CallClassStoreTextTestCase( - description="no relations passed yet", - context_token=None, - prior_relations=(), - expected_equal=False, - ), - CallClassStoreTextTestCase( - description="one relation passed", - context_token=None, - prior_relations=(SqlResourceRef(kind=SqlResourceRefKind.MODEL, name="orders"),), - expected_equal=False, - ), - CallClassStoreTextTestCase( - description="context macro", - context_token="ctx", - prior_relations=None, - expected_equal=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_call_class_inputs_when_naming_it_then_only_identical_inputs_match( - test_case: CallClassStoreTextTestCase, -) -> None: - plain: str | None = call_class_store_text( - macro_tokens=("cents",), context_token=None, prior_relations=None - ) - - text: str | None = call_class_store_text( - macro_tokens=("cents",), - context_token=test_case.context_token, - prior_relations=test_case.prior_relations, - ) - - assert (text == plain) is test_case.expected_equal diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py index bcaccfe10c..0534a84871 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/classes/test_macro_bridge.py @@ -6,8 +6,7 @@ import pytest -from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge -from sqlbuild.compiler.macro_bridge.models import MacroCallScan +import sqlbuild._native as _native from tests.unit.src.sqlbuild.compiler.macro_bridge.classes._test_types import ( DeepNestingScanTestCase, ) @@ -37,20 +36,18 @@ ], ids=lambda case: case.description, ) -def test_given_deeply_nested_calls_when_scanning_then_the_bridge_returns_promptly( +def test_given_deeply_nested_calls_when_scanning_then_the_native_scan_returns_promptly( test_case: DeepNestingScanTestCase, ) -> None: - bridge: MacroBridge = MacroBridge( - python_version=(sys.version_info[0], sys.version_info[1]), - unicode_version=unicodedata.unidata_version, - ) sql: str = "@m(" * test_case.depth + "x" + ")" * test_case.depth started: float = time.perf_counter() - scan: MacroCallScan = bridge.scan(sql) + rows, failure = _native.scan_macro_call_sites( + sql, (sys.version_info[0], sys.version_info[1]), unicodedata.unidata_version + ) assert time.perf_counter() - started < test_case.expected_max_seconds - assert (tuple(site.tree_names for site in scan.sites), scan.failure) == ( + assert (tuple(tuple(row[4] or ()) for row in rows), failure) == ( test_case.expected_tree_names, None, ) diff --git a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py index 401f3038df..34ac7d46ae 100644 --- a/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/macro_bridge/main/helpers.py @@ -52,7 +52,7 @@ def recorded() -> str: def _scan_with_active_bridge() -> None: bridge: MacroBridge = cast(MacroBridge, active_macro_bridge()) - _ = bridge.scan("SELECT @cents('amount') AS amount") + _ = bridge.stats() def recording_stage( diff --git a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py index ce4a3c9bc1..a459ffea88 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/_test_types.py @@ -1,17 +1,16 @@ from dataclasses import dataclass, field from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledObjectKey, - CompiledProject, ) +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput @dataclass(frozen=True) class ManifestTopLevelTestCase: description: str - project: CompiledProject + project: CompiledProjectView plan_output: PlanOutput project_name: str adapter_type: str @@ -28,7 +27,7 @@ class ManifestTopLevelTestCase: @dataclass(frozen=True) class ManifestModelNodeTestCase: description: str - model: CompiledModel + model: CompiledModelView plan_entries: tuple[ModelPlanEntry, ...] project_name: str expected_unique_id: str diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index cf9d71491f..b689ed3aae 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -20,7 +20,13 @@ CompileModelConfig, LoadedMacro, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompiledSeedView, + CompiledSourceView, +) from sqlbuild.compiler.discovery.models import ( DiscoveredSchemaFile, DiscoveredSeedFile, @@ -101,7 +107,7 @@ def build_test_model( description: str | None = None, columns: tuple[SchemaColumn, ...] = (), meta: dict[str, object] | None = None, -) -> CompiledModel: +) -> CompiledModelView: """Build a minimal CompiledModel for manifest tests.""" effective_alias: str = cast(str, (name, alias)[alias is not None]) @@ -145,7 +151,7 @@ def build_test_source( table: str | None = None, description: str | None = None, columns: tuple[SourceColumnEntry, ...] = (), -) -> CompiledSource: +) -> CompiledSourceView: """Build a minimal CompiledSource for manifest tests.""" entry: SourceEntry = SourceEntry( @@ -176,7 +182,7 @@ def build_test_seed( database: str | None = None, schema: str | None = "public", tags: tuple[str, ...] = (), -) -> CompiledSeed: +) -> CompiledSeedView: """Build a minimal CompiledSeed for manifest tests.""" return CompiledSeed( @@ -230,12 +236,12 @@ def build_test_plan_entry( def build_test_project( *, - models: tuple[CompiledModel, ...] = (), - sources: tuple[CompiledSource, ...] = (), - seeds: tuple[CompiledSeed, ...] = (), + models: tuple[CompiledModelView, ...] = (), + sources: tuple[CompiledSourceView, ...] = (), + seeds: tuple[CompiledSeedView, ...] = (), sql_hook_files: tuple[DiscoveredSqlHookFile, ...] = (), loaded_macros: dict[str, LoadedMacro] | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """Build a minimal CompiledProject for manifest tests.""" return CompiledProject( diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 77a59535d7..485555ada0 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -7,13 +7,16 @@ import pytest from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledObjectKey, - CompiledProject, - CompiledSeed, - CompiledSource, LoadedMacro, ) +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledSeedView, + CompiledSourceView, +) +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile, SqlHookEntry from sqlbuild.compiler.manifest._helpers.shared import build_fqn from sqlbuild.compiler.manifest.constants import DBT_MANIFEST_SCHEMA_VERSION @@ -59,7 +62,7 @@ _PROJECT: str = "demo" _ADAPTER: str = "duckdb" -_MODEL_ORDERS: CompiledModel = build_test_model( +_MODEL_ORDERS: CompiledModelView = build_test_model( name="orders", query_sql="SELECT * FROM raw", relative_path="models/staging/orders.sql", @@ -69,14 +72,14 @@ columns=(SchemaColumn(name="id", type="INTEGER"),), ) -_MODEL_ORPHAN: CompiledModel = build_test_model( +_MODEL_ORPHAN: CompiledModelView = build_test_model( name="orphan", query_sql="SELECT 1", relative_path="models/orphan.sql", schema="public", ) -_MODEL_WITH_COLUMNS: CompiledModel = build_test_model( +_MODEL_WITH_COLUMNS: CompiledModelView = build_test_model( name="typed_model", relative_path="models/typed_model.sql", description="Has columns", @@ -86,13 +89,13 @@ ), ) -_MODEL_WITH_DEPS: CompiledModel = build_test_model( +_MODEL_WITH_DEPS: CompiledModelView = build_test_model( name="joined", relative_path="models/joined.sql", deps=(model_key("orders"), source_key("raw_customers")), ) -_MODEL_WITH_ALIAS: CompiledModel = build_test_model( +_MODEL_WITH_ALIAS: CompiledModelView = build_test_model( name="orders_aliased", relative_path="models/marts/orders_aliased.sql", database="analytics", @@ -102,7 +105,7 @@ config_values={"materialized": "incremental", "tags": ["core", "nightly"]}, ) -_SOURCE_WITH_COLUMNS: CompiledSource = build_test_source( +_SOURCE_WITH_COLUMNS: CompiledSourceView = build_test_source( name="raw_events", database="raw_db", schema="events", @@ -114,14 +117,14 @@ ), ) -_SOURCE_RAW_ORDERS: CompiledSource = build_test_source( +_SOURCE_RAW_ORDERS: CompiledSourceView = build_test_source( name="raw_orders", database="raw_db", schema="public", table="orders_table", description="Raw order data", ) -_SOURCE_EVENTS: CompiledSource = build_test_source( +_SOURCE_EVENTS: CompiledSourceView = build_test_source( name="events", database=None, schema="analytics", @@ -422,7 +425,7 @@ def test_given_project_when_building_manifest_then_produces_correct_structure( def test_given_model_when_building_manifest_then_produces_correct_node( test_case: ManifestModelNodeTestCase, ) -> None: - project: CompiledProject = build_test_project(models=(test_case.model,)) + project: CompiledProjectView = build_test_project(models=(test_case.model,)) plan_output: PlanOutput = build_test_plan_output(model_entries=test_case.plan_entries) result: dict[str, Any] = run_manifest( @@ -512,8 +515,8 @@ def test_given_model_when_building_manifest_then_produces_correct_node( def test_given_source_when_building_manifest_then_produces_correct_node( test_case: ManifestSourceNodeTestCase, ) -> None: - source: CompiledSource = _SOURCE_FIXTURES[test_case.expected_source_name] - project: CompiledProject = build_test_project(sources=(source,)) + source: CompiledSourceView = _SOURCE_FIXTURES[test_case.expected_source_name] + project: CompiledProjectView = build_test_project(sources=(source,)) plan_output: PlanOutput = build_test_plan_output() result: dict[str, Any] = run_manifest( @@ -559,8 +562,8 @@ def test_given_source_when_building_manifest_then_produces_correct_node( def test_given_seed_when_building_manifest_then_produces_correct_node( test_case: ManifestSeedNodeTestCase, ) -> None: - seed: CompiledSeed = build_test_seed(name="country_codes", tags=test_case.expected_tags) - project: CompiledProject = build_test_project(seeds=(seed,)) + seed: CompiledSeedView = build_test_seed(name="country_codes", tags=test_case.expected_tags) + project: CompiledProjectView = build_test_project(seeds=(seed,)) plan_output: PlanOutput = build_test_plan_output() result: dict[str, Any] = run_manifest( @@ -637,9 +640,9 @@ def test_given_relative_path_when_building_fqn_then_returns_expected_parts( def test_given_deps_when_building_manifest_then_parent_map_correct( test_case: ManifestParentMapTestCase, ) -> None: - source: CompiledSource = build_test_source(name="raw_orders") - model: CompiledModel = build_test_model(name="orders", deps=(source_key("raw_orders"),)) - project: CompiledProject = build_test_project(models=(model,), sources=(source,)) + source: CompiledSourceView = build_test_source(name="raw_orders") + model: CompiledModelView = build_test_model(name="orders", deps=(source_key("raw_orders"),)) + project: CompiledProjectView = build_test_project(models=(model,), sources=(source,)) plan_output: PlanOutput = build_test_plan_output( model_entries=(build_test_plan_entry(name="orders"),), ) @@ -744,11 +747,11 @@ def test_given_macro_when_building_manifest_then_macro_node_present( raw_source=f"def {dependency_name}(): return 'SELECT 1'", function=lambda: "SELECT 1", ) - project: CompiledProject = build_test_project( + project: CompiledProjectView = build_test_project( loaded_macros=loaded_macros, ) - project = replace( - project, + project = project_with( + project=project, scope_index=ScopeIndex(declarations=(declaration, *dependency_records)), ) plan_output: PlanOutput = build_test_plan_output() @@ -796,8 +799,8 @@ def test_given_macro_when_building_manifest_then_macro_node_present( def test_given_audit_when_building_manifest_then_produces_test_node( test_case: ManifestAuditNodeTestCase, ) -> None: - model: CompiledModel = build_test_model(name="orders") - project: CompiledProject = build_test_project(models=(model,)) + model: CompiledModelView = build_test_model(name="orders") + project: CompiledProjectView = build_test_project(models=(model,)) audit_entry: AuditPlanEntry = build_test_audit_plan_entry( name="not_null_orders_id", resolved_sql="SELECT id FROM staging.orders WHERE id IS NULL", @@ -851,8 +854,8 @@ def test_given_audit_when_building_manifest_then_produces_test_node( def test_given_sql_test_when_building_manifest_then_produces_test_node( test_case: ManifestSqlTestNodeTestCase, ) -> None: - model: CompiledModel = build_test_model(name="orders") - project: CompiledProject = build_test_project(models=(model,)) + model: CompiledModelView = build_test_model(name="orders") + project: CompiledProjectView = build_test_project(models=(model,)) chain: tuple[ChainStep, ...] = ( ChainStep(model_name="stg_orders", resolved_sql=""), ChainStep( @@ -933,7 +936,7 @@ def test_given_full_project_when_building_manifest_then_validates_against_dbt_sc ) schema: dict[str, Any] = json.loads(schema_path.read_text(encoding="utf-8")) - model: CompiledModel = build_test_model( + model: CompiledModelView = build_test_model( name="orders", query_sql="SELECT id FROM raw", relative_path="models/staging/orders.sql", @@ -943,14 +946,14 @@ def test_given_full_project_when_building_manifest_then_validates_against_dbt_sc columns=(SchemaColumn(name="id", type="INTEGER"),), deps=(source_key("raw_orders"),), ) - source: CompiledSource = build_test_source( + source: CompiledSourceView = build_test_source( name="raw_orders", database="raw_db", schema="public", table="orders", ) - seed: CompiledSeed = build_test_seed(name="codes") - project: CompiledProject = build_test_project( + seed: CompiledSeedView = build_test_seed(name="codes") + project: CompiledProjectView = build_test_project( models=(model,), sources=(source,), seeds=(seed,), @@ -1010,7 +1013,7 @@ def test_given_full_project_when_building_manifest_then_validates_against_dbt_sc function=lambda: "SELECT 1", ) - project = replace(project, loaded_macros={"my_macro": macro}) + project = project_with(project=project, loaded_macros={"my_macro": macro}) result: dict[str, Any] = run_manifest( project=project, plan_output=plan_output, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index e508eda85f..1083d400ef 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -12,7 +12,11 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.fingerprints.models import Fingerprint @@ -22,12 +26,12 @@ def build_single_model_project( logical_database: str | None, physical_schema: str | None, physical_database: str | None, -) -> CompiledProject: +) -> CompiledProjectView: """Build a project with one model for deferred target testing.""" qualified: str | None = (None, f"{physical_schema}.test_model")[physical_schema is not None] - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="test_model"), deps=(), name="test_model", diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py index 90b58bc559..6bcaee331c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py @@ -13,7 +13,7 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType class ConservativeCustomAdapter(BaseAdapter): @@ -35,7 +35,7 @@ def _execute(self, connection: Any, sql: str) -> Any: def build_project( *, target: CompiledRelationLocation, effective_connection: dict[str, object] -) -> CompiledProject: +) -> CompiledProjectView: """Build a minimal compiled project for target validation tests.""" return CompiledProject( diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py index 3e7fe83a1f..14ffe833c9 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/test_main.py @@ -13,9 +13,9 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.models import ( - CompiledProject, CompiledRelationLocation, ) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSourceFile from sqlbuild.compiler.pipeline._helpers.target_validation import ( validate_managed_loader_target_isolation, @@ -112,7 +112,7 @@ def test_given_resolved_clone_connection_when_validating_loaders_then_origin_cre def test_given_adapter_with_missing_target_namespace_when_validating_then_raises_clear_error( test_case: ValidateProjectTargetsTestCase, ) -> None: - project: CompiledProject = build_project(target=test_case.target, effective_connection={}) + project: CompiledProjectView = build_project(target=test_case.target, effective_connection={}) with pytest.raises(ValueError, match=test_case.expected_error_fragment): validate_project_targets(adapter_name=test_case.adapter_name, project=project) @@ -138,7 +138,7 @@ def test_given_adapter_with_missing_target_namespace_when_validating_then_raises def test_given_duckdb_with_missing_target_namespace_when_validating_then_it_passes( test_case: ValidateProjectTargetsTestCase, ) -> None: - project: CompiledProject = build_project(target=test_case.target, effective_connection={}) + project: CompiledProjectView = build_project(target=test_case.target, effective_connection={}) validate_project_targets(adapter_name=test_case.adapter_name, project=project) assert test_case.expected_error_fragment == "" @@ -195,7 +195,7 @@ def test_given_duckdb_with_missing_target_namespace_when_validating_then_it_pass def test_given_warehouse_managed_write_without_schema_when_validating_then_error_is_raised( test_case: ValidateManagedWriteSchemaTestCase, ) -> None: - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( target=CompiledRelationLocation( database=None, schema=test_case.target_schema, @@ -239,7 +239,7 @@ def test_given_warehouse_managed_write_without_schema_when_validating_then_error def test_given_explicit_or_permitted_schema_when_validating_managed_write_then_it_passes( test_case: ValidateManagedWriteSchemaTestCase, ) -> None: - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( target=CompiledRelationLocation( database=None, schema=test_case.target_schema, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py index 3e538cd117..6fe6d27209 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/test_deferred_locations.py @@ -7,9 +7,9 @@ from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( - CompiledProject, CompiledRelationLocation, ) +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline._helpers.deferred_locations import build_deferred_locations from sqlbuild.spec.contracts.models import TargetConfig from tests.unit.src.sqlbuild.compiler.pipeline._helpers._test_types import ( @@ -127,7 +127,7 @@ def test_given_deferred_target_config_when_building_targets_then_resolves_expected_naming( test_case: DeferredTargetTestCase, ) -> None: - project: CompiledProject = build_single_model_project( + project: CompiledProjectView = build_single_model_project( logical_schema=test_case.logical_schema, logical_database=test_case.logical_database, physical_schema=test_case.logical_schema or test_case.default_schema, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index 176eddab20..f3d84efc7b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -10,7 +10,7 @@ CompiledRelationLocation, CompiledSource, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason @@ -36,7 +36,7 @@ def relation_target_python_node() -> None: """No-op Python declaration used by relation target graph tests.""" -def build_relation_target_project() -> CompiledProject: +def build_relation_target_project() -> CompiledProjectView: source_entry: SourceEntry = SourceEntry(name="orders", schema="load_raw", table="orders") source_file: DiscoveredSourceFile = DiscoveredSourceFile( file_path=Path("sources/raw.yml"), diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 98da979be6..3629e4b28c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -7,6 +7,7 @@ import pytest from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline.main import _compile_phase as compile_phase_module from sqlbuild.compiler.pipeline.main import compile as compile_module from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult @@ -48,7 +49,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di monkeypatch: pytest.MonkeyPatch, ) -> None: monotonic: Mock = Mock(side_effect=test_case.clock_values) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="timing-run", effective_target_name="test", effective_connection={}, diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py index e138a1c049..d4e855c383 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_relation_targets.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.pipeline.main.relation_targets import build_python_relation_targets from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph @@ -37,7 +37,7 @@ def test_given_source_read_map_when_building_python_relation_targets_then_uses_read_relation( test_case: PythonRelationTargetsTestCase, ) -> None: - project: CompiledProject = build_relation_target_project() + project: CompiledProjectView = build_relation_target_project() raw_source: SourceEntry = project.sources[0].source_entry read_source: SourceEntry = SourceEntry(name="orders", schema="deferred_raw", table="orders") plan_output: PlanOutput = PlanOutput( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index e294ba57b9..9348e7321a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -8,11 +8,10 @@ ) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, CompileSqlReference, InferredColumn, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledProjectView from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ( CursorBounds, @@ -386,24 +385,6 @@ class ExpandDownstreamTestCase: expected_keys: frozenset[CompiledObjectKey] -@dataclass(frozen=True) -class FindPathKeysTestCase: - description: str - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - start: CompiledObjectKey - end: CompiledObjectKey - expected_keys: frozenset[CompiledObjectKey] - - -@dataclass(frozen=True) -class FindPathKeysErrorTestCase: - description: str - downstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] - start: CompiledObjectKey - end: CompiledObjectKey - expected_error_type: type[Exception] - - @dataclass(frozen=True) class ParseSelectorTestCase: description: str @@ -672,7 +653,7 @@ class ScenarioExecutionPlanTestCase: class ScenarioUnmockedSeedExecutionPlanTestCase: description: str graph_plan: ScenarioGraphPlan - project: CompiledProject + project: CompiledProjectView expected_seed_fixture_names: frozenset[str] expected_seed_entry_targets: dict[str, str] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index 0352763fc1..7e983c8db4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -17,7 +17,13 @@ CompileModelConfig, FunctionArgument, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + FunctionLanguage, +) from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.identity._version_identity_metadata import ( @@ -34,7 +40,7 @@ _STUB_TS: datetime = datetime(2026, 1, 15, 12, 0, 0) -def build_model_from_test_case(test_case: DetectModelChangesTestCase) -> CompiledModel: +def build_model_from_test_case(test_case: DetectModelChangesTestCase) -> CompiledModelView: """Build a CompiledModel from a test case.""" schema_entry: SchemaModelEntry | None = _build_schema_entry(test_case) @@ -56,7 +62,7 @@ def build_model_from_test_case(test_case: DetectModelChangesTestCase) -> Compile def build_model_from_metadata_test_case( test_case: DetectModelMetadataTestCase, -) -> CompiledModel: +) -> CompiledModelView: return CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), deps=tuple( @@ -131,12 +137,12 @@ def build_metadata_json_with_audit_gate(metadata_json: str) -> str: return json.dumps(metadata, sort_keys=True, separators=(",", ":"), default=str) -def build_project_for_function_metadata_detection() -> CompiledProject: +def build_project_for_function_metadata_detection() -> CompiledProjectView: function_key: CompiledObjectKey = CompiledObjectKey( resource_type=CompiledResourceType.UDF, name="is_large_order", ) - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), deps=(function_key,), name="orders", @@ -153,7 +159,7 @@ def build_project_for_function_metadata_detection() -> CompiledProject: name="is_large_order", qualified_name=None, ) - function: CompiledFunction = CompiledFunction( + function: CompiledFunctionView = CompiledFunction( key=function_key, deps=(), name="is_large_order", @@ -184,7 +190,7 @@ def build_scope_for_function_metadata_detection() -> PlannerScope: resource_type=CompiledResourceType.UDF, name="is_large_order", ) - project: CompiledProject = build_project_for_function_metadata_detection() + project: CompiledProjectView = build_project_for_function_metadata_detection() return PlannerScope( upstream_deps={model_key: (function_key,), function_key: ()}, downstream_deps={function_key: (model_key,), model_key: ()}, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 2b18caf1fa..b8c8eb6343 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -5,7 +5,8 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledModel, InferredColumn +from sqlbuild.compiler.compile.models import InferredColumn +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.models import Fingerprint @@ -361,7 +362,7 @@ def test_given_model_and_snapshot_when_detecting_changes_then_returns_expected( test_case: DetectModelChangesTestCase, ) -> None: - model: CompiledModel = build_model_from_test_case(test_case) + model: CompiledModelView = build_model_from_test_case(test_case) snapshot: WarehouseSnapshot = build_snapshot_from_test_case(test_case) result: ChangeDetectionResult = detect_model_changes( @@ -433,7 +434,7 @@ def test_given_dropped_relation_with_recorded_build_when_planning_then_plans_fir expected_change_kind=ChangeKind.FIRST_RUN, expected_backfill_action=BackfillAction.FULL, ) - model: CompiledModel = build_model_from_test_case(detection_case) + model: CompiledModelView = build_model_from_test_case(detection_case) change: ChangeDetectionResult = detect_model_changes( model=model, @@ -547,7 +548,7 @@ def test_given_dropped_relation_with_recorded_build_when_planning_then_plans_fir def test_given_model_identity_metadata_when_detecting_changes_then_uses_aligned_metadata( test_case: DetectModelMetadataTestCase, ) -> None: - model: CompiledModel = build_model_from_metadata_test_case(test_case) + model: CompiledModelView = build_model_from_metadata_test_case(test_case) snapshot: WarehouseSnapshot = build_snapshot_for_metadata_test_case(test_case) result: ChangeDetectionResult = detect_model_changes( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py index 03c3a06fb1..cb9ef4d121 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/conftest.py @@ -4,15 +4,13 @@ from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, ) +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views from sqlbuild.compiler.planner._helpers.graph.core import ( build_downstream_deps, build_execution_upstream_deps, ) -from sqlbuild.compiler.planner.main.selection._build_model_path_index import ( - build_model_path_index, -) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( build_test_project, model_key, @@ -25,7 +23,7 @@ def diamond_graph() -> tuple[ dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], ]: - project: CompiledProject = build_test_project( + project: CompiledProjectView = build_test_project( model_deps={ "orders": ("raw_orders",), "customers": ("raw_customers",), @@ -59,7 +57,7 @@ def path_graph() -> tuple[ dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], dict[CompiledObjectKey, str], ]: - project: CompiledProject = build_test_project( + project: CompiledProjectView = build_test_project( model_deps={ "stg_orders": ("raw_orders",), "stg_customers": ("raw_customers",), @@ -83,5 +81,5 @@ def path_graph() -> tuple[ upstream ) all_keys: dict[str, CompiledObjectKey] = {key.name: key for key in upstream} - path_idx: dict[CompiledObjectKey, str] = build_model_path_index(project) + path_idx: dict[CompiledObjectKey, str] = project_lineage_views(project).path_index return all_keys, upstream, downstream, path_idx diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 98b314f050..9453017c0c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -3,9 +3,10 @@ from __future__ import annotations import hashlib -from collections.abc import Callable +from collections.abc import Callable, Iterator from dataclasses import replace from datetime import UTC, datetime +from itertools import chain from pathlib import Path from types import MappingProxyType from typing import ClassVar @@ -14,9 +15,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile._helpers.sql_tests.core import ( - extract_assertion_target_model_names, -) +from sqlbuild.compiler.compile._helpers.refs.references import scan_sql_reference_calls from sqlbuild.compiler.compile.models import ( CompiledAudit, CompiledDirectLogicSqlTestPayload, @@ -38,9 +37,17 @@ ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, + CompiledAuditView, + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, CompiledResourceType, + CompiledSeedView, + CompiledSourceView, SqlTestMode, ) +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredAuditBlock, DiscoveredAuditFile, @@ -100,7 +107,7 @@ def microbatch_model_with_cursor_end( test_case: MicrobatchCursorEndPlanTestCase, -) -> CompiledModel: +) -> CompiledModelView: """Build a microbatch model with a configured terminal cursor bound.""" return CompiledModel( @@ -244,7 +251,7 @@ def build_selection_model( key: CompiledObjectKey, name: str, materialized: str, -) -> CompiledModel: +) -> CompiledModelView: """Build a minimal model for selection helper tests.""" values: dict[str, object] = {"materialized": materialized} @@ -278,7 +285,7 @@ def load_orders(_ctx: object) -> list[dict[str, object]]: return [] -def build_source_load_nodes_project() -> CompiledProject: +def build_source_load_nodes_project() -> CompiledProjectView: raw_orders_entry: SourceEntry = SourceEntry( name="raw_orders", table="orders", @@ -355,7 +362,7 @@ def build_test_project( sql_test_expected_model_names: tuple[str, ...] = (), table_fn_test_function_names: tuple[str, ...] = (), audit_model_source_deps: dict[str, tuple[str, ...]] | None = None, -) -> CompiledProject: +) -> CompiledProjectView: """Build a minimal CompiledProject for graph tests.""" effective_paths: dict[str, str] = model_paths or {} @@ -363,7 +370,7 @@ def build_test_project( seed_name_set: set[str] = set(seed_names) dbt_ref_name_set: set[str] = set(dbt_ref_names) function_name_set: set[str] = set(function_names) - models: list[CompiledModel] = [] + models: list[CompiledModelView] = [] model_name: str dep_names: tuple[str, ...] for model_name, dep_names in (model_deps or {}).items(): @@ -392,7 +399,7 @@ def build_test_project( ) ) - sources: list[CompiledSource] = [] + sources: list[CompiledSourceView] = [] source_name: str for source_name in source_names: source_entry: SourceEntry = SourceEntry( @@ -413,7 +420,7 @@ def build_test_project( ) ) - seeds: list[CompiledSeed] = [] + seeds: list[CompiledSeedView] = [] seed_name: str for seed_name in seed_names: seeds.append( @@ -433,7 +440,7 @@ def build_test_project( ) ) - functions: list[CompiledFunction] = [] + functions: list[CompiledFunctionView] = [] function_name: str for function_name in function_names: functions.append( @@ -514,7 +521,7 @@ def build_test_project( ) sql_tests.extend(((), (table_function_sql_test,))[bool(table_fn_test_function_names)]) - audits: list[CompiledAudit] = [] + audits: list[CompiledAuditView] = [] model_name: str audit_source_names: tuple[str, ...] for model_name, audit_source_names in (audit_model_source_deps or {}).items(): @@ -588,8 +595,8 @@ def build_scenario_from_test_case( CompileSqlScenarioCte(name=f"__expected__{model_name}", sql_body="SELECT 1") for model_name in test_case.expected_model_names ) - assertion_target_model_names: tuple[str, ...] = extract_assertion_target_model_names( - assertion_sql=test_case.assertion_sql_bodies, syntax=_GENERIC_SQL_SYNTAX + assertion_target_model_names: tuple[str, ...] = _assertion_ref_targets( + test_case.assertion_sql_bodies ) return CompiledSqlScenario( key=scenario_key, @@ -644,7 +651,7 @@ def build_scenario_cli_identifier_limit_pipeline( ) -> tuple[CompiledSqlScenario, CompilePipelineResult]: """Build a minimal pipeline result for CLI scenario identifier-limit tests.""" - project: CompiledProject = build_test_project( + project: CompiledProjectView = build_test_project( model_deps={model_name: ("raw__orders",)}, source_names=("raw__orders",), ) @@ -678,7 +685,7 @@ def build_scenario_cli_identifier_limit_pipeline( source_fixture_names=("raw__orders",), expected_model_names=(model_name,), ) - project = replace(project, sql_scenarios=(scenario,)) + project = project_with(project=project, sql_scenarios=(scenario,)) return scenario, CompilePipelineResult(project=project, plan_output=PlanOutput()) @@ -712,10 +719,10 @@ def build_scenario_relation_test_map() -> ScenarioRelationMap: ) -def build_scenario_relation_test_project() -> CompiledProject: +def build_scenario_relation_test_project() -> CompiledProjectView: """Build a project covering scenario relation planning tests.""" - project: CompiledProject = build_test_project( + project: CompiledProjectView = build_test_project( model_deps={ "daily_revenue": ("raw__orders", "stg_customers", "country_codes"), "customer_revenue": ("raw__orders",), @@ -724,9 +731,9 @@ def build_scenario_relation_test_project() -> CompiledProject: source_names=("raw__orders",), seed_names=("country_codes",), ) - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} models_by_name["daily_revenue"] = replace( - models_by_name["daily_revenue"], + compiled_model_record(models_by_name["daily_revenue"]), query_sql=( 'SELECT * FROM __source("raw__orders") ' 'JOIN __ref("stg_customers") USING (customer_id) ' @@ -734,16 +741,16 @@ def build_scenario_relation_test_project() -> CompiledProject: 'JOIN __dbt_ref("stripe", "payments") USING (customer_id)' ), ) - models: list[CompiledModel] = [models_by_name[model.name] for model in project.models] - return replace(project, models=tuple(models)) + models: list[CompiledModelView] = [models_by_name[model.name] for model in project.models] + return project_with(project=project, models=tuple(models)) -def build_scenario_relation_test_project_with_unused_seed() -> CompiledProject: +def build_scenario_relation_test_project_with_unused_seed() -> CompiledProjectView: """Build a scenario relation test project with a seed outside the scenario graph.""" - project: CompiledProject = build_scenario_relation_test_project() - unused_seed_project: CompiledProject = build_test_project(seed_names=("unused_seed",)) - return replace(project, seeds=(*project.seeds, unused_seed_project.seeds[0])) + project: CompiledProjectView = build_scenario_relation_test_project() + unused_seed_project: CompiledProjectView = build_test_project(seed_names=("unused_seed",)) + return project_with(project=project, seeds=(*project.seeds, unused_seed_project.seeds[0])) def build_scenario_relation_test_scenario( @@ -815,10 +822,10 @@ def build_scenario_relation_test_scenario( ) -def build_test_project_with_source_entry(source_entry: SourceEntry) -> CompiledProject: +def build_test_project_with_source_entry(source_entry: SourceEntry) -> CompiledProjectView: """Build a minimal CompiledProject containing one source entry.""" - source: CompiledSource = CompiledSource( + source: CompiledSourceView = CompiledSource( key=source_key(source_entry.name), deps=(), name=source_entry.name, @@ -841,7 +848,7 @@ def build_test_project_with_source_entry(source_entry: SourceEntry) -> CompiledP def build_source_cursor_input_model( test_case: SourceCursorInputColumnsTestCase, -) -> CompiledModel: +) -> CompiledModelView: """Build an incremental model for source cursor input column validation.""" config_values: dict[str, object] = ( @@ -873,10 +880,10 @@ def build_source_cursor_input_model( def build_transitive_watermark_models( *, watermark_name: str, watermark_column: str -) -> tuple[CompiledModel, dict[str, CompiledModel]]: +) -> tuple[CompiledModelView, dict[str, CompiledModelView]]: """Build a consumer, direct view, and transitive source watermark lineage.""" - view_model: CompiledModel = CompiledModel( + view_model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders_view"), deps=(), name="orders_view", @@ -893,7 +900,7 @@ def build_transitive_watermark_models( qualified_name="staging.orders_view", ), ) - consumer: CompiledModel = CompiledModel( + consumer: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="test_model"), deps=(), name="test_model", @@ -925,11 +932,11 @@ def build_transitive_watermark_models( def build_function_transitive_watermark_models( *, watermark_name: str, watermark_column: str -) -> tuple[CompiledModel, dict[str, CompiledModel], dict[str, CompiledFunction]]: +) -> tuple[CompiledModelView, dict[str, CompiledModelView], dict[str, CompiledFunctionView]]: """Build a consumer whose physical watermark is upstream of a table function.""" - consumer: CompiledModel - models_by_name: dict[str, CompiledModel] + consumer: CompiledModelView + models_by_name: dict[str, CompiledModelView] consumer, models_by_name = build_transitive_watermark_models( watermark_name=watermark_name, watermark_column=watermark_column, @@ -941,7 +948,7 @@ def build_function_transitive_watermark_models( name=function_name, qualified_name=f"analytics.{function_name}", ) - function: CompiledFunction = CompiledFunction( + function: CompiledFunctionView = CompiledFunction( key=CompiledObjectKey(resource_type=CompiledResourceType.TABLE_FN, name=function_name), deps=(), name=function_name, @@ -954,7 +961,7 @@ def build_function_transitive_watermark_models( references=(CompileSqlReference(ref_kind=SqlReferenceKind.REF, ref_name="orders_view"),), ) consumer = replace( - consumer, + compiled_model_record(consumer), query_sql=f'SELECT * FROM __table_fn("{function_name}")()', references=( CompileSqlReference( @@ -968,8 +975,8 @@ def build_function_transitive_watermark_models( def build_cursor_input_contract_models( test_case: SourceCursorInputColumnsTestCase, -) -> dict[str, CompiledModel]: - models: dict[str, CompiledModel] = { +) -> dict[str, CompiledModelView]: + models: dict[str, CompiledModelView] = { test_case.reference_name: CompiledModel( key=CompiledObjectKey( resource_type=CompiledResourceType.MODEL, @@ -1043,7 +1050,7 @@ def _resolve_dep_key( return CompiledObjectKey(resource_type=resource_type, name=name) -def build_strategy_model(test_case: ResolveModelPlanActionTestCase) -> CompiledModel: +def build_strategy_model(test_case: ResolveModelPlanActionTestCase) -> CompiledModelView: """Build a CompiledModel from an action resolution test case.""" config_values: dict[str, object] = ( @@ -1087,7 +1094,7 @@ def build_strategy_change_result( ) -def build_strategy_error_model(test_case: IncrementalStrategyErrorTestCase) -> CompiledModel: +def build_strategy_error_model(test_case: IncrementalStrategyErrorTestCase) -> CompiledModelView: """Build a CompiledModel from a strategy error test case.""" config_values: dict[str, object] = {"materialized": test_case.materialized} | ( @@ -1140,7 +1147,7 @@ def build_warnings_change_result( def build_audit_from_test_case( test_case: PlanAuditTestCase, -) -> CompiledAudit: +) -> CompiledAuditView: """Build a minimal CompiledAudit from a test case.""" return CompiledAudit( @@ -1204,10 +1211,10 @@ def build_cursor_ref(ref_name: str) -> CompileSqlReference: def build_cursor_model_map( ref_name: str, qualified_name: str | None, -) -> dict[str, CompiledModel]: +) -> dict[str, CompiledModelView]: """Build a model map with one entry for cursor resolution tests.""" - model_map: dict[str, CompiledModel] = { + model_map: dict[str, CompiledModelView] = { ref_name: CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=ref_name), deps=(), @@ -1243,7 +1250,7 @@ def build_cursor_deferred_locations( return (None, locations)[qualified_name is not None] -def build_cursor_override_model(cursor_type: str | None) -> CompiledModel: +def build_cursor_override_model(cursor_type: str | None) -> CompiledModelView: """Build a minimal model with optional cursor_type for override resolution tests.""" config_values: dict[str, object] = {"materialized": "incremental"} | ( @@ -1271,7 +1278,7 @@ def build_microbatch_lookback_model( incremental_strategy: str, batch_size: str, lookback: str | None, -) -> CompiledModel: +) -> CompiledModelView: """Build a microbatch model for lookback resolution tests.""" config_values: dict[str, object] = { @@ -1360,7 +1367,7 @@ def build_scheduling_audit( references: tuple[CompileSqlReference, ...], attached_target_kind: AttachedAuditTargetKind | None, attached_target_name: str | None, -) -> CompiledAudit: +) -> CompiledAuditView: """Build a minimal CompiledAudit for scheduling tests.""" stub_file: DiscoveredAuditFile = DiscoveredAuditFile( @@ -1423,7 +1430,7 @@ def build_seed_identity_compiled_seed( file_path: Path, *, csv_settings: SeedCsvSettings | None = None, -) -> CompiledSeed: +) -> CompiledSeedView: resolved_csv_settings: SeedCsvSettings = csv_settings or SeedCsvSettings() schema_entry: SchemaSeedEntry = SchemaSeedEntry( name="orders", csv_settings=resolved_csv_settings @@ -1476,3 +1483,16 @@ def native_audit_scope_deps( CompiledObjectKey(resource_type=CompiledResourceType(kind), name=name) for kind, name in audit_deps[0] ) + + +def _assertion_ref_targets(assertion_sql: tuple[str, ...]) -> tuple[str, ...]: + """The unique `__ref` targets the assertion bodies read, in authored order.""" + + references: Iterator[CompileSqlReference] = chain.from_iterable( + scan_sql_reference_calls(sql=sql, syntax=_GENERIC_SQL_SYNTAX).references + for sql in assertion_sql + ) + refs: Iterator[CompileSqlReference] = filter( + lambda reference: reference.ref_kind == SqlReferenceKind.REF, references + ) + return tuple(dict.fromkeys(reference.ref_name for reference in refs)) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index b9d4a799f8..9092c3b15c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -16,7 +16,7 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.migrations.columns import plan_column_migrations from sqlbuild.compiler.planner.exceptions import PlannerInputError @@ -49,7 +49,7 @@ def test_given_untokenizable_tracked_model_when_planning_column_migrations_then_error_names_model( test_case: UnfingerprintableCandidateTestCase, ) -> None: - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), deps=(), name="orders", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index 4fd899ae11..98414f42c7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -13,7 +13,11 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.planner.models import ( PlannerRelationsContext, PlannerRuntime, @@ -44,7 +48,7 @@ def build_retention_planner_inputs( table_type_downgrade: TableTypeDowngradePolicy = TableTypeDowngradePolicy.REQUIRE_CONFIRMATION, model_names: tuple[str, ...] = ("orders",), ) -> tuple[PlannerRuntime, PlannerWarehouseState, PlannerScope]: - models: tuple[CompiledModel, ...] = tuple( + models: tuple[CompiledModelView, ...] = tuple( _retention_model( name=name, desired_days=desired_days, @@ -54,7 +58,7 @@ def build_retention_planner_inputs( for name in model_names ) keys: tuple[CompiledObjectKey, ...] = tuple(model.key for model in models) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="run-1", effective_target_name="test", effective_connection={}, @@ -121,7 +125,7 @@ def _retention_model( desired_days: int, config_values: dict[str, object], table_type: ResolvedTableType | None, -) -> CompiledModel: +) -> CompiledModelView: return CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name), deps=(), @@ -146,7 +150,7 @@ def _retention_model( ) -def build_full_refresh_model(*, config_values: dict[str, object]) -> CompiledModel: +def build_full_refresh_model(*, config_values: dict[str, object]) -> CompiledModelView: """Build an incremental model with the given header configuration.""" return CompiledModel( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py index e6f37aefd9..d5460e3284 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_scopes.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes from sqlbuild.compiler.planner.models import ( PlannerPolicies, @@ -64,7 +64,7 @@ def test_given_selection_when_resolving_scopes_then_stale_warning_scope_is_upstream_closure( test_case: PlannerScopesTestCase, ) -> None: - project: CompiledProject = build_test_project(model_deps=test_case.model_deps) + project: CompiledProjectView = build_test_project(model_deps=test_case.model_deps) resolution: PlannerScopeResolution = resolve_planner_scopes( project=project, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 96ae0645d0..500efbbcb9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -11,11 +11,11 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner.models import ModelPlanContext -def build_cursor_intrinsic_model(*, config_values: dict[str, object]) -> CompiledModel: +def build_cursor_intrinsic_model(*, config_values: dict[str, object]) -> CompiledModelView: """Build a cursor incremental model that projects both cursor intrinsics.""" return CompiledModel( @@ -69,7 +69,7 @@ def render_table_function_call(self, *, target: str, call_suffix_sql: str) -> st def build_cursor_models( *, model_cursor_types: tuple[tuple[str, str | None], ...] -) -> tuple[CompiledModel, ...]: +) -> tuple[CompiledModelView, ...]: """Build models named by `model_cursor_types`; a None type builds a non-cursor table.""" return tuple( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py index e49767fc62..1ac4c873f5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_cursor_intrinsics.py @@ -3,7 +3,7 @@ import pytest from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_sql from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -66,7 +66,7 @@ def test_given_cursor_interval_when_resolving_then_renders_expected_sql( test_case: CursorIntrinsicResolutionTestCase, ) -> None: - model: CompiledModel = build_cursor_intrinsic_model(config_values=test_case.model_config) + model: CompiledModelView = build_cursor_intrinsic_model(config_values=test_case.model_config) resolved_sql: str = resolve_model_sql( adapter=DuckDbAdapter(), @@ -104,7 +104,7 @@ def test_given_cursor_interval_when_resolving_then_renders_expected_sql( def test_given_missing_cursor_interval_when_resolving_then_rejects_intrinsics( test_case: CursorIntrinsicResolutionTestCase, ) -> None: - model: CompiledModel = build_cursor_intrinsic_model(config_values=test_case.model_config) + model: CompiledModelView = build_cursor_intrinsic_model(config_values=test_case.model_config) with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): resolve_model_sql( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py index b9b7b95d33..c71184bd0b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_hooks.py @@ -7,7 +7,7 @@ import pytest from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_hook_entries from sqlbuild.compiler.planner.models import ModelPlanContext @@ -64,7 +64,7 @@ def test_given_sql_hook_references_when_planning_then_they_resolve_like_model_sql( test_case: HookSqlResolutionTestCase, ) -> None: - model: CompiledModel = build_cursor_intrinsic_model( + model: CompiledModelView = build_cursor_intrinsic_model( config_values={"pre_hooks": test_case.pre_hooks, "post_hooks": test_case.post_hooks} ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 03f3af6119..b559748ef1 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -34,7 +34,12 @@ LoadedMacro, MacroContext, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.discovery.models import ( DiscoveredSqlTestBlock, DiscoveredSqlTestFile, @@ -82,7 +87,7 @@ def plan_single_test( *, test: CompiledSqlTest, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, sql_analysis_enabled: bool = False, ) -> tuple[SqlTestPlanEntry, tuple[PlanWarning, ...]]: @@ -102,7 +107,7 @@ def plan_single_test( def plan_single_test_allowing_errors( *, test: CompiledSqlTest, - project: CompiledProject, + project: CompiledProjectView, adapter: BaseAdapter, sql_analysis_enabled: bool = False, ) -> tuple[SqlTestPlanResult, NativeSqlTestPlan]: @@ -128,7 +133,7 @@ def plan_single_test_allowing_errors( def assert_native_artifact_matches_runtime_plan( *, - project: CompiledProject, + project: CompiledProjectView, sql_test: CompiledSqlTest, entry: SqlTestPlanEntry, warnings: tuple[PlanWarning, ...], @@ -166,7 +171,7 @@ def assert_native_artifact_matches_runtime_plan( def build_test_and_project( test_case: PlanTestChainTestCase, -) -> tuple[CompiledSqlTest, CompiledProject]: +) -> tuple[CompiledSqlTest, CompiledProjectView]: """Build a CompiledSqlTest and CompiledProject from a test case.""" authored_ctes: list[CompileSqlTestCte] = [] @@ -263,7 +268,7 @@ def build_test_and_project( ), ) - models: list[CompiledModel] = [] + models: list[CompiledModelView] = [] model_name: str query_sql: str for model_name, query_sql in test_case.model_queries.items(): @@ -304,7 +309,7 @@ def build_test_and_project( ) ) - functions: list[CompiledFunction] = [] + functions: list[CompiledFunctionView] = [] function_name: str qualified_name: str for function_name, qualified_name in test_case.function_locations.items(): @@ -362,7 +367,7 @@ def build_test_and_project( ) ) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="test_run", effective_target_name=None, effective_connection={}, @@ -506,7 +511,7 @@ def _build_test_sql_body( } -def build_cursor_model(*, cursor_type: str, cursor_grain: str | None) -> CompiledModel: +def build_cursor_model(*, cursor_type: str, cursor_grain: str | None) -> CompiledModelView: """Build one incremental model whose query uses both cursor intrinsics.""" values: dict[str, object] = { diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 789aedcede..769f17f68d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -19,7 +19,8 @@ CompiledSqlTest, CompileSqlTestCte, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( plan_compiled_sql_test_artifacts, @@ -484,7 +485,7 @@ def test_given_test_and_project_when_planning_then_produces_expected_chain( test_case: PlanTestChainTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) entry: SqlTestPlanEntry @@ -627,7 +628,7 @@ def test_given_unresolved_chain_input_when_planning_then_test_is_blocked_with_er test_case: PlanTestChainTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) result: SqlTestPlanResult @@ -724,7 +725,7 @@ def test_given_model_chain_when_native_planning_then_artifact_matches_runtime_sq test_case: NativePlanningDifferentialTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case.planning_case) assert isinstance(compiled_test.payload, CompiledModelSqlTestPayload) compiled_test = replace( @@ -740,7 +741,7 @@ def test_given_model_chain_when_native_planning_then_artifact_matches_runtime_sq ), ), ) - project = replace(project, sql_tests=(compiled_test,)) + project = project_with(project=project, sql_tests=(compiled_test,)) adapter: DuckDbAdapter = DuckDbAdapter() entry, warnings = plan_single_test( test=compiled_test, @@ -800,7 +801,7 @@ def test_given_refs_in_comments_and_literals_when_planning_then_preserves_sql_an test_case: PlanTestChainTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) entry, warnings = plan_single_test(test=compiled_test, project=project, adapter=DuckDbAdapter()) @@ -840,7 +841,7 @@ def test_given_repeated_query_with_different_fixture_rows_when_planning_then_kee expected_cte_bodies={"orders": f"SELECT {fixture_id} AS id"}, ) compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(fixture_case) entry, warnings = plan_single_test( test=compiled_test, @@ -879,7 +880,7 @@ def test_given_unresolved_marker_in_mock_when_planning_with_analysis_then_report test_case: PlanTestChainTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) result: SqlTestPlanResult @@ -1038,7 +1039,7 @@ def test_given_udf_sql_test_when_planning_then_compares_resolved_actual_to_expec ), ) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="test_run", effective_target_name=None, effective_connection={}, @@ -1169,7 +1170,7 @@ def test_given_table_function_sql_test_when_planning_then_compares_resolved_actu ), ) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="test_run", effective_target_name=None, effective_connection={}, @@ -1275,7 +1276,7 @@ def test_given_sql_analysis_enabled_when_planning_test_then_it_uses_top_level_ge test_case: PlanTestChainTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) entry: SqlTestPlanEntry @@ -1328,7 +1329,7 @@ def test_given_sql_analysis_enabled_when_generated_cte_name_conflicts_then_it_ra test_case: PlanTestChainTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) with pytest.raises(ValueError, match=test_case.expected_error_fragments[0]): @@ -1372,7 +1373,7 @@ def test_given_sql_analysis_assertion_when_planning_then_uses_shared_chain_ctes( test_case: PlanTestChainTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) entry: SqlTestPlanEntry @@ -1443,7 +1444,7 @@ def test_given_adapter_function_when_resolving_with_analysis_then_keeps_authored test_case: SqlAnalysisDialectTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project( PlanTestChainTestCase( description=test_case.description, @@ -1515,7 +1516,7 @@ def test_given_tsql_shared_cte_name_when_planning_then_helper_and_model_cte_stay test_case: PlanTestChainTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) adapter: BaseAdapter = build_comparison_test_adapter("sqlserver") @@ -1602,7 +1603,7 @@ def test_given_reference_the_compiler_missed_when_planning_then_it_fails_with_lo test_case: PlanTestChainTestCase, ) -> None: compiled_test: CompiledSqlTest - project: CompiledProject + project: CompiledProjectView compiled_test, project = build_test_and_project(test_case) with pytest.raises(SqlTestReferenceError) as raised: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index 953c0a520b..a88912efab 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -4,12 +4,11 @@ from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditRunScope from sqlbuild.compiler.compile.models import ( - CompiledAudit, CompiledObjectKey, CompiledRelationLocation, CompileSqlReference, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledAuditView from sqlbuild.compiler.planner._helpers.output.audit_entry import plan_audit from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.compiler.references.types import SqlReferenceKind @@ -57,7 +56,7 @@ def test_given_logically_attached_audit_when_planning_then_scheduling_and_identi CompileSqlReference(ref_kind=SqlReferenceKind.REF, ref_name=name) for name in test_case.referenced_model_names ) - audit: CompiledAudit = build_scheduling_audit( + audit: CompiledAuditView = build_scheduling_audit( references=references, attached_target_kind=AttachedAuditTargetKind.MODEL, attached_target_name=test_case.attached_target_name, @@ -142,7 +141,7 @@ def test_given_logically_attached_audit_when_planning_then_scheduling_and_identi def test_given_audit_when_planning_then_resolves_sql( test_case: PlanAuditTestCase, ) -> None: - audit: CompiledAudit = build_audit_from_test_case(test_case) + audit: CompiledAuditView = build_audit_from_test_case(test_case) model_locations: dict[str, CompiledRelationLocation] = build_audit_model_locations( test_case.model_locations ) @@ -180,7 +179,7 @@ def test_given_audit_when_planning_then_resolves_sql( def test_given_audit_with_unresolved_marker_when_planning_then_it_raises_clear_error( test_case: PlanAuditTestCase, ) -> None: - audit: CompiledAudit = build_audit_from_test_case(test_case) + audit: CompiledAuditView = build_audit_from_test_case(test_case) model_locations: dict[str, CompiledRelationLocation] = build_audit_model_locations( test_case.model_locations ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py index 68673e9352..cbdff45f0a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_audit_scheduling.py @@ -9,12 +9,12 @@ AuditRunScope, ) from sqlbuild.compiler.compile.models import ( - CompiledAudit, CompiledObjectKey, CompileSqlReference, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, + CompiledAuditView, ) from sqlbuild.compiler.planner._helpers.output.audit_scheduling import ( reads_outside_target_lineage, @@ -203,7 +203,7 @@ def test_given_audit_refs_when_resolving_attachment_then_returns_expected( test_case: ResolveAttachmentTestCase, ) -> None: - audit: CompiledAudit = build_scheduling_audit( + audit: CompiledAuditView = build_scheduling_audit( references=test_case.references, attached_target_kind=test_case.attached_target_kind, attached_target_name=test_case.attached_target_name, @@ -241,7 +241,7 @@ def test_given_audit_refs_when_resolving_attachment_then_returns_expected( def test_given_invalid_attachment_when_resolving_then_raises( test_case: ResolveAttachmentErrorTestCase, ) -> None: - audit: CompiledAudit = build_scheduling_audit( + audit: CompiledAuditView = build_scheduling_audit( references=test_case.references, attached_target_kind=test_case.attached_target_kind, attached_target_name=test_case.attached_target_name, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py index d71bdddb9f..41cd712f21 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_cursor_override_resolution.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.output.plan_entry import resolve_cursor_overrides from sqlbuild.compiler.planner.models import CursorOverrides from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( @@ -84,7 +84,7 @@ def test_given_model_and_overrides_when_resolving_then_returns_expected( test_case: CursorOverrideResolutionTestCase, ) -> None: - model: CompiledModel = build_cursor_override_model(test_case.cursor_type) + model: CompiledModelView = build_cursor_override_model(test_case.cursor_type) has_typed: bool = any( v is not None for v in (test_case.start_ts, test_case.end_ts, test_case.start_int, test_case.end_int) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 47549d22bd..9e44554ac2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -6,13 +6,17 @@ import pytest from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledObjectKey, - CompiledProject, CompiledSqlTest, CompileSqlReference, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner._helpers.fixtures import completion as fixture_completion from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner._helpers.sql_tests import assembly as sql_test_assembly @@ -72,8 +76,8 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is ref_kind=SqlReferenceKind.SOURCE, ref_name="raw_orders", ) - model: CompiledModel = replace( - project.models[0], + model: CompiledModelView = replace( + compiled_model_record(project.models[0]), references=(source_reference,), ) tests: tuple[CompiledSqlTest, ...] = tuple( @@ -87,8 +91,8 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is ) for index in range(test_case.test_count) ) - test_project: CompiledProject = replace( - project, + test_project: CompiledProjectView = project_with( + project=project, models=(model,), sql_tests=tests, settings=SettingsConfig(sql_analysis=True), @@ -211,7 +215,9 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ ref_kind=SqlReferenceKind.SOURCE, ref_name="raw_orders", ) - model: CompiledModel = replace(project.models[0], references=(source_reference,)) + model: CompiledModelView = replace( + compiled_model_record(project.models[0]), references=(source_reference,) + ) source_key: tuple[CompiledResourceType, str] = ( CompiledResourceType.SOURCE, "raw_orders", @@ -246,7 +252,7 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ ) completed: RelationFixtureCompletion = fixture_completion.build_relation_fixture_completion( - project=replace(project, models=(model,)), + project=project_with(project=project, models=(model,)), adapter=PlannerTestAdapter(), ordered_model_names=(model.name,), fixture_groups=((CompiledResourceType.SOURCE, {"raw_orders": test_case.fixture_sql}),), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py index 1cd0ea0875..0a1d58915f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_function_fingerprints.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledFunction +from sqlbuild.compiler.compile.types import CompiledFunctionView from sqlbuild.compiler.planner._helpers.identity.functions import ( build_compiled_function_fingerprint_sql, detect_function_change, @@ -65,7 +65,7 @@ def test_given_function_fingerprint_when_detecting_change_then_returns_reason( test_case: DetectFunctionChangeTestCase, ) -> None: - function: CompiledFunction = build_compiled_function( + function: CompiledFunctionView = build_compiled_function( body_sql=test_case.body_sql, target_schema=test_case.target_schema, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py index e9253459cc..f68ec6b046 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_graph.py @@ -9,7 +9,6 @@ build_execution_upstream_deps, expand_downstream, expand_upstream, - find_path_keys, topologically_order_keys, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( @@ -19,8 +18,6 @@ ExecutionEdgeOriginsTestCase, ExpandDownstreamTestCase, ExpandUpstreamTestCase, - FindPathKeysErrorTestCase, - FindPathKeysTestCase, SqlTestFunctionGraphDepsTestCase, TopologicalOrderTestCase, ) @@ -341,73 +338,6 @@ def test_given_key_when_expanding_downstream_then_returns_expected_keys( assert result == test_case.expected_keys -@pytest.mark.parametrize( - "test_case", - [ - FindPathKeysTestCase( - description="finds all nodes on directed paths between start and end", - downstream={ - source_key("raw"): (model_key("a"),), - model_key("a"): (model_key("b"), model_key("c")), - model_key("b"): (model_key("d"),), - model_key("c"): (model_key("d"),), - model_key("d"): (), - }, - start=source_key("raw"), - end=model_key("d"), - expected_keys=frozenset( - { - source_key("raw"), - model_key("a"), - model_key("b"), - model_key("c"), - model_key("d"), - } - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_start_and_end_when_finding_path_keys_then_returns_expected( - test_case: FindPathKeysTestCase, -) -> None: - result: frozenset[CompiledObjectKey] = find_path_keys( - start=test_case.start, - end=test_case.end, - downstream=test_case.downstream, - ) - - assert result == test_case.expected_keys - - -@pytest.mark.parametrize( - "test_case", - [ - FindPathKeysErrorTestCase( - description="raises when end is not downstream of start", - downstream={ - model_key("a"): (model_key("b"),), - model_key("b"): (), - model_key("c"): (), - }, - start=model_key("a"), - end=model_key("c"), - expected_error_type=ValueError, - ), - ], - ids=lambda case: case.description, -) -def test_given_unreachable_end_when_finding_path_keys_then_raises( - test_case: FindPathKeysErrorTestCase, -) -> None: - with pytest.raises(test_case.expected_error_type, match="not downstream"): - find_path_keys( - start=test_case.start, - end=test_case.end, - downstream=test_case.downstream, - ) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py index f166b9b0f3..351663804b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_lookback.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.output.plan_entry import resolve_microbatch_lookback from sqlbuild.compiler.planner.types import IncrementalStrategy from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( @@ -60,7 +60,7 @@ def test_given_microbatch_model_when_resolving_lookback_then_returns_expected_value( test_case: MicrobatchLookbackTestCase, ) -> None: - model: CompiledModel = build_microbatch_lookback_model( + model: CompiledModelView = build_microbatch_lookback_model( incremental_strategy=test_case.incremental_strategy, batch_size=test_case.batch_size, lookback=test_case.lookback, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index be821f1347..f793f116ce 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -13,7 +13,7 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner._helpers.output.plan_entry import ( _compute_microbatch_range, _compute_plan_cursor_bounds, @@ -74,7 +74,7 @@ def test_given_planner_owned_microbatch_with_cursor_end_when_planning_then_sentinels_are_deferred( test_case: MicrobatchCursorEndPlanTestCase, ) -> None: - model: CompiledModel = microbatch_model_with_cursor_end(test_case) + model: CompiledModelView = microbatch_model_with_cursor_end(test_case) snapshot: WarehouseSnapshot = WarehouseSnapshot( cursor_snapshots={ model.name: ModelCursorSnapshot( @@ -138,7 +138,7 @@ def test_given_planner_owned_microbatch_with_cursor_end_when_planning_then_senti def test_given_explicit_cursor_overrides_without_snapshot_when_planning_then_uses_overrides( test_case: PlanEntryCursorOverrideTestCase, ) -> None: - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), deps=(), name="orders", @@ -213,7 +213,7 @@ def test_given_explicit_cursor_overrides_without_snapshot_when_planning_then_use def test_given_runtime_owned_missing_snapshot_when_overrides_complete_then_all_ranges_match( test_case: AuthoritativeCursorOverrideTestCase, ) -> None: - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), deps=(), name="orders", @@ -247,7 +247,7 @@ def test_given_runtime_owned_missing_snapshot_when_overrides_complete_then_all_r } ) - microbatch_model: CompiledModel = replace( + microbatch_model: CompiledModelView = replace( model, config=CompileModelConfig( values={ @@ -319,7 +319,7 @@ def test_given_runtime_owned_missing_snapshot_when_overrides_complete_then_all_r def test_given_planner_owned_model_backed_input_when_grain_is_coarser_then_range_is_concrete( test_case: PlannerOwnedMixedGrainRangeTestCase, ) -> None: - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="hourly_context"), deps=(), name="hourly_context", @@ -397,7 +397,7 @@ def test_given_planner_owned_model_backed_input_when_grain_is_coarser_then_range def test_given_snapshot_cursor_grain_when_planning_then_dml_bound_matches_query_interval( test_case: PlanEntryCursorGrainTestCase, ) -> None: - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), deps=(), name="orders", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index 37192101cb..208e7ebed7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -13,10 +13,11 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.models import ( - CompiledProject, CompiledSqlScenario, CompileSqlScenarioCte, ) +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner._helpers.scenario.relations import ( build_scenario_execution_plan, @@ -391,13 +392,13 @@ def test_given_project_source_ref_in_scenario_fixture_when_building_fixture_plan def test_given_scenario_graph_when_building_execution_plan_then_returns_scenario_plan( test_case: ScenarioExecutionPlanTestCase, ) -> None: - base_project: CompiledProject = build_scenario_relation_test_project() + base_project: CompiledProjectView = build_scenario_relation_test_project() def notify() -> None: return None - project: CompiledProject = replace( - base_project, + project: CompiledProjectView = project_with( + project=base_project, functions=( build_compiled_function( body_sql=( @@ -512,7 +513,7 @@ def notify() -> None: def test_given_required_unmocked_seed_when_building_execution_plan_then_loads_project_seed( test_case: ScenarioUnmockedSeedExecutionPlanTestCase, ) -> None: - project: CompiledProject = test_case.project + project: CompiledProjectView = test_case.project relation_plan: ScenarioRelationPlan = build_scenario_relation_plan( project=project, graph_plan=test_case.graph_plan, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index 8fff444c99..69fef260e4 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -6,8 +6,13 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey, CompiledProject -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes @@ -66,7 +71,7 @@ def test_given_complete_change_results_when_building_stale_warnings_then_models_ test_case: CompleteModelChangesWorkTestCase, monkeypatch: pytest.MonkeyPatch, ) -> None: - model: CompiledModel = build_selection_model( + model: CompiledModelView = build_selection_model( key=MODEL_KEY, name=MODEL_KEY.name, materialized="table", @@ -180,33 +185,37 @@ def test_given_partial_selection_when_resolving_scopes_then_inspects_only_upstre resource_type=CompiledResourceType.MODEL, name="unrelated", ) - root: CompiledModel = build_selection_model( + root: CompiledModelView = build_selection_model( key=root_key, name=root_key.name, materialized="table", ) - middle: CompiledModel = replace( - build_selection_model( - key=middle_key, - name=middle_key.name, - materialized="table", + middle: CompiledModelView = replace( + compiled_model_record( + build_selection_model( + key=middle_key, + name=middle_key.name, + materialized="table", + ) ), deps=(root_key,), ) - leaf: CompiledModel = replace( - build_selection_model( - key=leaf_key, - name=leaf_key.name, - materialized="table", + leaf: CompiledModelView = replace( + compiled_model_record( + build_selection_model( + key=leaf_key, + name=leaf_key.name, + materialized="table", + ) ), deps=(middle_key,), ) - unrelated: CompiledModel = build_selection_model( + unrelated: CompiledModelView = build_selection_model( key=unrelated_key, name=unrelated_key.name, materialized="table", ) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="run", effective_target_name=None, effective_connection={}, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py index 78918712c3..5d2a749719 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selectors.py @@ -4,8 +4,9 @@ from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, ) +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps from sqlbuild.compiler.planner._helpers.graph.selectors import ( parse_selector, @@ -706,15 +707,11 @@ def test_given_invalid_path_selector_when_resolving_then_raises_with_message( def test_given_model_paths_when_building_path_index_then_returns_expected_folders( test_case: BuildPathIndexTestCase, ) -> None: - from sqlbuild.compiler.planner.main.selection._build_model_path_index import ( - build_model_path_index, - ) - - project: CompiledProject = build_test_project( + project: CompiledProjectView = build_test_project( model_deps={name: () for name in test_case.model_paths}, model_paths=test_case.model_paths, ) - result: dict[CompiledObjectKey, str] = build_model_path_index(project) + result: dict[CompiledObjectKey, str] = project_lineage_views(project).path_index result_by_name: dict[str, str] = {key.name: folder for key, folder in result.items()} assert result_by_name == test_case.expected_folders diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 8311a2054c..2137d7e8b7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -10,12 +10,15 @@ from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, CompiledObjectKey, CompiledProject, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledFunctionView, + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.planner._helpers.output.plan_entry import ( _build_cursor_input_relations, _runtime_cursor_producer_names, @@ -156,7 +159,7 @@ def get_columns_for_relations( def test_given_sources_when_gathering_columns_then_returns_expected_source_columns( test_case: SourceColumnsTestCase, ) -> None: - project: CompiledProject = build_test_project_with_source_entry(test_case.source_entry) + project: CompiledProjectView = build_test_project_with_source_entry(test_case.source_entry) adapter: _RecordingAdapter = _RecordingAdapter(test_case.adapter_column_names) result: dict[str, tuple[ColumnInfo, ...]] = gather_source_columns( @@ -186,7 +189,7 @@ def test_given_sources_when_gathering_columns_then_returns_expected_source_colum def test_given_multi_database_sources_when_gathering_columns_then_matches_current_database_only( test_case: MultiDatabaseSourceColumnsTestCase, ) -> None: - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="test", effective_target_name=None, effective_connection={}, @@ -237,7 +240,7 @@ def test_given_multi_database_sources_when_gathering_columns_then_matches_curren def test_given_known_source_columns_when_building_relations_context_then_reuses_prior_gather( test_case: KnownSourceColumnsReuseTestCase, ) -> None: - project: CompiledProject = build_test_project_with_source_entry( + project: CompiledProjectView = build_test_project_with_source_entry( SourceEntry( name="raw_payments", expression="SELECT 1 AS id, 1700 AS amount_cents, 'success' AS status", @@ -402,7 +405,7 @@ def test_given_known_source_columns_when_building_relations_context_then_reuses_ def test_given_valid_cursor_input_source_columns_when_validating_then_passes( test_case: SourceCursorInputColumnsTestCase, ) -> None: - model: CompiledModel = build_source_cursor_input_model(test_case) + model: CompiledModelView = build_source_cursor_input_model(test_case) source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] = {} for source_name, column_names in test_case.source_columns.items(): columns: list[ColumnInfo] = [] @@ -436,13 +439,13 @@ def test_given_valid_cursor_input_source_columns_when_validating_then_passes( def test_given_view_filter_and_transitive_watermark_when_planning_then_relation_is_physical( test_case: WatermarkLineageTestCase, ) -> None: - model: CompiledModel - models_by_name: dict[str, CompiledModel] + model: CompiledModelView + models_by_name: dict[str, CompiledModelView] model, models_by_name = build_transitive_watermark_models( watermark_name=test_case.watermark_name, watermark_column=test_case.watermark_column, ) - view_model: CompiledModel = models_by_name["orders_view"] + view_model: CompiledModelView = models_by_name["orders_view"] relations: tuple[CursorInputRelation, ...] = _build_cursor_input_relations( model=model, @@ -521,7 +524,7 @@ def test_given_view_filter_and_transitive_watermark_when_planning_then_relation_ def test_given_missing_cursor_input_source_column_when_validating_then_raises_clear_error( test_case: SourceCursorInputColumnsTestCase, ) -> None: - model: CompiledModel = build_source_cursor_input_model(test_case) + model: CompiledModelView = build_source_cursor_input_model(test_case) source_warehouse_columns: dict[str, tuple[ColumnInfo, ...]] = {} for source_name, column_names in test_case.source_columns.items(): columns: list[ColumnInfo] = [] @@ -557,8 +560,8 @@ def test_given_missing_cursor_input_source_column_when_validating_then_raises_cl def test_given_transitive_watermark_when_validating_lineage_then_accepts_upstream_source( test_case: WatermarkLineageTestCase, ) -> None: - model: CompiledModel - models_by_name: dict[str, CompiledModel] + model: CompiledModelView + models_by_name: dict[str, CompiledModelView] model, models_by_name = build_transitive_watermark_models( watermark_name=test_case.watermark_name, watermark_column=test_case.watermark_column, @@ -589,9 +592,9 @@ def test_given_transitive_watermark_when_validating_lineage_then_accepts_upstrea def test_given_table_function_and_transitive_watermark_when_planning_then_source_is_accepted( test_case: WatermarkLineageTestCase, ) -> None: - model: CompiledModel - models_by_name: dict[str, CompiledModel] - functions_by_name: dict[str, CompiledFunction] + model: CompiledModelView + models_by_name: dict[str, CompiledModelView] + functions_by_name: dict[str, CompiledFunctionView] model, models_by_name, functions_by_name = build_function_transitive_watermark_models( watermark_name=test_case.watermark_name, watermark_column=test_case.watermark_column, @@ -646,8 +649,8 @@ def test_given_table_function_and_transitive_watermark_when_planning_then_source def test_given_invalid_watermark_when_validating_lineage_then_raises_precise_error( test_case: WatermarkLineageTestCase, ) -> None: - model: CompiledModel - models_by_name: dict[str, CompiledModel] + model: CompiledModelView + models_by_name: dict[str, CompiledModelView] model, models_by_name = build_transitive_watermark_models( watermark_name=test_case.watermark_name, watermark_column=test_case.watermark_column, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index 11ac5cbf33..0773889e39 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -14,7 +14,12 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( build_intermediate_source_map, expand_selected_loader_dependencies, @@ -77,7 +82,7 @@ def test_given_selected_source_load_keys_when_building_entries_then_returns_ordered_load_nodes( test_case: SourceLoadNodesTestCase, ) -> None: - project: CompiledProject = build_source_load_nodes_project() + project: CompiledProjectView = build_source_load_nodes_project() selected_keys: frozenset[CompiledObjectKey] = frozenset( entry.key for entry in test_case.expected_entries ) @@ -113,7 +118,7 @@ def test_given_selected_source_load_keys_when_building_entries_then_returns_orde def test_given_unrelated_selected_seed_when_building_source_map_then_sources_are_excluded( test_case: SourceMetadataClosureTestCase, ) -> None: - project: CompiledProject = build_source_load_nodes_project() + project: CompiledProjectView = build_source_load_nodes_project() seed_key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.SEED, "country_codes") source_map: dict[str, SourceEntry] = build_source_load_map( @@ -138,7 +143,7 @@ def test_given_unrelated_selected_seed_when_building_source_map_then_sources_are def test_given_transitive_unselected_upstream_when_building_source_map_then_source_is_retained( test_case: SourceMetadataClosureTestCase, ) -> None: - project: CompiledProject = build_source_load_nodes_project() + project: CompiledProjectView = build_source_load_nodes_project() selected_key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.MODEL, "fact_orders") upstream_key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.MODEL, "stg_orders") source_key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.SOURCE, "raw_orders") @@ -171,7 +176,7 @@ def test_given_unselected_upstream_in_closure_when_filtering_metadata_then_uses_ ) -> None: selected_key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.MODEL, "fact_orders") upstream_key: CompiledObjectKey = CompiledObjectKey(CompiledResourceType.MODEL, "stg_orders") - selected_model: CompiledModel = CompiledModel( + selected_model: CompiledModelView = CompiledModel( key=selected_key, deps=(upstream_key,), name="fact_orders", @@ -185,7 +190,7 @@ def test_given_unselected_upstream_in_closure_when_filtering_metadata_then_uses_ qualified_name="ANALYTICS_DB.ANALYTICS.FACT_ORDERS_PHYSICAL", ), ) - upstream_model: CompiledModel = CompiledModel( + upstream_model: CompiledModelView = CompiledModel( key=upstream_key, deps=(), name="stg_orders", @@ -199,7 +204,7 @@ def test_given_unselected_upstream_in_closure_when_filtering_metadata_then_uses_ qualified_name="ANALYTICS_DB.STAGING.STG_ORDERS_PHYSICAL", ), ) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="test", effective_target_name=None, effective_connection={}, @@ -249,7 +254,7 @@ def test_given_unselected_upstream_in_closure_when_filtering_metadata_then_uses_ def test_given_terminal_source_when_expanding_loader_deps_then_adds_intermediate_source( test_case: LoaderDagExpansionTestCase, ) -> None: - project: CompiledProject = build_source_load_nodes_project() + project: CompiledProjectView = build_source_load_nodes_project() selected_keys: frozenset[CompiledObjectKey] = frozenset( CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=name) for name in test_case.selected_names @@ -301,9 +306,9 @@ def test_given_terminal_source_when_expanding_loader_deps_then_adds_intermediate def test_given_qualified_intermediate_destination_when_building_map_then_parses_loader_target( test_case: LoaderDestinationPlanningTestCase, ) -> None: - project: CompiledProject = build_source_load_nodes_project() - project = replace( - project, + project: CompiledProjectView = build_source_load_nodes_project() + project = project_with( + project=project, effective_target_database="default_db", effective_target_schema="default_schema", loader_functions=( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py index b87f55b136..176bf94ee9 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_strategy_action.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledModel +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.output.strategy import resolve_model_plan_action from sqlbuild.compiler.planner.models import ChangeDetectionResult, SchemaFinding from sqlbuild.compiler.planner.types import ( @@ -323,7 +323,7 @@ def test_given_model_and_changes_when_resolving_action_then_returns_expected( test_case: ResolveModelPlanActionTestCase, ) -> None: - model: CompiledModel = build_strategy_model(test_case) + model: CompiledModelView = build_strategy_model(test_case) change_result: ChangeDetectionResult = build_strategy_change_result(test_case) action: PlanAction @@ -363,7 +363,7 @@ def test_given_model_and_changes_when_resolving_action_then_returns_expected( def test_given_incremental_with_bad_strategy_when_resolving_action_then_raises( test_case: IncrementalStrategyErrorTestCase, ) -> None: - model: CompiledModel = build_strategy_error_model(test_case) + model: CompiledModelView = build_strategy_error_model(test_case) change_result: ChangeDetectionResult = build_strategy_error_change_result(test_case) with pytest.raises(test_case.expected_error_type, match=test_case.expected_error_fragment): diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_resolution.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_resolution.py index c1cbd8c19e..bd2850f379 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_resolution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_resolution.py @@ -5,10 +5,10 @@ import pytest from sqlbuild.compiler.compile.models import ( - CompiledModel, CompiledRelationLocation, CompileSqlReference, ) +from sqlbuild.compiler.compile.types import CompiledModelView from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( _resolve_upstream_qualified_name, ) @@ -65,7 +65,7 @@ def test_given_ref_and_deferred_locations_when_resolving_upstream_then_returns_e test_case: CursorUpstreamResolutionTestCase, ) -> None: ref: CompileSqlReference = build_cursor_ref(test_case.ref_name) - model_map: dict[str, CompiledModel] = build_cursor_model_map( + model_map: dict[str, CompiledModelView] = build_cursor_model_map( test_case.ref_name, test_case.model_qualified_name, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index 427f15cb71..140cd1ade2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -12,7 +12,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSource -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledResourceType, CompiledSourceView from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.compiler.planner._helpers.warehouse.snapshot import ( _CursorModelInfo, @@ -177,7 +177,7 @@ def check_one_source_table( source_key: CompiledObjectKey = CompiledObjectKey( resource_type=CompiledResourceType.SOURCE, name=source.name ) - compiled_source: CompiledSource = CompiledSource( + compiled_source: CompiledSourceView = CompiledSource( key=source_key, deps=(), name=source.name, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py index 880fecbf85..00f0fa9c35 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -13,7 +13,7 @@ from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.pipeline.main.project import compile_project from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import ( @@ -59,7 +59,7 @@ def test_given_multiple_missing_source_schemas_when_inspecting_then_metadata_is_ }, ) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter ) listings: Mock = Mock(wraps=adapter.list_relations) @@ -116,7 +116,7 @@ def test_given_source_schema_case_differs_when_inspecting_then_columns_are_found }, ) adapter: DuckDbAdapter = DuckDbAdapter() - project: CompiledProject = compile_project( + project: CompiledProjectView = compile_project( discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter ) with duckdb.connect() as connection: diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index f6d3a5e589..6e0dc98a6b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -14,7 +14,7 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType from sqlbuild.compiler.planner.main.execution.display_plan import ( build_display_only_sqlbuild_plan, ) @@ -58,7 +58,7 @@ def test_given_incremental_override_when_building_display_plan_then_uses_effective_refresh( test_case: DisplayFullRefreshTestCase, ) -> None: - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), deps=(), name="orders", diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index ae828ac29e..fd9b29e292 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -13,7 +13,11 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint @@ -31,13 +35,13 @@ from sqlbuild.spec.contracts.models import SchemaColumn, SchemaModelEntry -def model_definition_hash(project: CompiledProject, name: str) -> str: - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} - model: CompiledModel = models_by_name[name] +def model_definition_hash(project: CompiledProjectView, name: str) -> str: + models_by_name: dict[str, CompiledModelView] = {model.name: model for model in project.models} + model: CompiledModelView = models_by_name[name] return compute_query_hash(query_sql=model.query_sql, dialect=project.sql_analysis_dialect) -def build_sqlbuild_model_selector_project() -> CompiledProject: +def build_sqlbuild_model_selector_project() -> CompiledProjectView: return CompiledProject( run_id="selector-test-run", effective_target_name=None, @@ -65,7 +69,7 @@ def build_sqlbuild_model_selector_project() -> CompiledProject: def build_sqlbuild_model_selector_model( *, name: str, relative_path: Path, tags: tuple[str, ...] -) -> CompiledModel: +) -> CompiledModelView: return CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name), deps=(), @@ -102,10 +106,10 @@ def grouped(model: type) -> dict[str, Any]: ) -def build_protected_schema_replay_project() -> CompiledProject: +def build_protected_schema_replay_project() -> CompiledProjectView: """Build a full_refresh false incremental whose declared column forces a full replay.""" - model: CompiledModel = CompiledModel( + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="order_history"), deps=(), name="order_history", diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index e4905e0d1d..57ca0ac495 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -9,7 +9,8 @@ CompileModelConfig, FunctionReturnColumn, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps from sqlbuild.compiler.planner._helpers.identity.direct import ( build_direct_model_version_identities, @@ -34,12 +35,12 @@ def build_table_function_graph_identities(*, base_query: str) -> DirectModelVers resource_type=CompiledResourceType.MODEL, name="customer_order_summary", ) - base_model: CompiledModel = _build_model( + base_model: CompiledModelView = _build_model( key=base_key, query_sql=base_query, deps=(), ) - consumer_model: CompiledModel = _build_model( + consumer_model: CompiledModelView = _build_model( key=consumer_key, query_sql='SELECT * FROM __table_fn("customer_orders")()', deps=(function_key,), @@ -76,7 +77,7 @@ def _build_model( key: CompiledObjectKey, query_sql: str, deps: tuple[CompiledObjectKey, ...], -) -> CompiledModel: +) -> CompiledModelView: return CompiledModel( key=key, deps=deps, @@ -94,11 +95,13 @@ def _build_model( def build_hooked_model_identity_json(*, hook_config: dict[str, object]) -> str: - model: CompiledModel = replace( - _build_model( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), - query_sql="SELECT 1 AS order_id", - deps=(), + model: CompiledModelView = replace( + compiled_model_record( + _build_model( + key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), + query_sql="SELECT 1 AS order_id", + deps=(), + ) ), config=CompileModelConfig(values=hook_config), ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py index 7f077e6953..3224a08207 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py @@ -1,6 +1,5 @@ from __future__ import annotations -from dataclasses import replace from pathlib import Path from typing import Any @@ -8,6 +7,8 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( @@ -123,8 +124,8 @@ def test_given_external_blocked_model_when_building_execution_plan_then_only_tha ) -> None: adapter: DuckDbAdapter = DuckDbAdapter() connection: Any = adapter.connect({"database": ":memory:"}) - project: CompiledProject = replace( - build_compiled_project_with_models( + project: CompiledProjectView = project_with( + project=build_compiled_project_with_models( { "blocked": "select 1 as id", "unrelated": "select 2 as id", diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 518f70c045..21a78e3899 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -9,7 +9,10 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject, CompiledSeed, CompileModelConfig +from sqlbuild.compiler.compile.models import CompileModelConfig +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSeedFile from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context @@ -44,15 +47,15 @@ def test_given_incremental_attached_audit_when_planning_then_no_warehouse_method test_case: StaticCommandPlanningTestCase, monkeypatch: pytest.MonkeyPatch, ) -> None: - project: CompiledProject = build_test_project( + project: CompiledProjectView = build_test_project( model_deps={"orders": ()}, audit_model_source_deps={"orders": ()}, ) - project = replace( - project, + project = project_with( + project=project, models=( replace( - project.models[0], + compiled_model_record(project.models[0]), config=CompileModelConfig(values={"materialized": "incremental", "cursor": "id"}), ), ), @@ -95,7 +98,7 @@ def test_given_incremental_attached_audit_when_planning_then_no_warehouse_method def test_given_sql_test_with_function_dependency_when_planning_then_required_function_is_projected( test_case: StaticCommandPlanningTestCase, ) -> None: - project: CompiledProject = build_test_project( + project: CompiledProjectView = build_test_project( model_deps={"orders": ("is_valid_order",)}, sql_test_expected_model_names=("orders",), function_names=("is_valid_order",), @@ -133,9 +136,9 @@ def test_given_selected_seed_when_planning_then_identity_and_reason_are_preserve ) -> None: seed_path: Path = tmp_path / "countries.csv" seed_path.write_text("code,name\nGB,United Kingdom\n", encoding="utf-8") - project: CompiledProject = build_test_project(seed_names=("countries",)) - project = replace( - project, + project: CompiledProjectView = build_test_project(seed_names=("countries",)) + project = project_with( + project=project, seeds=( replace( project.seeds[0], @@ -205,8 +208,8 @@ def test_given_selected_seed_without_state_table_when_reading_reason_then_only_s ) -> None: seed_path: Path = tmp_path / "countries.csv" seed_path.write_text("code\nGB\n", encoding="utf-8") - project: CompiledProject = build_test_project(seed_names=("countries",)) - seed: CompiledSeed = replace( + project: CompiledProjectView = build_test_project(seed_names=("countries",)) + seed: CompiledSeedView = replace( project.seeds[0], seed_file=DiscoveredSeedFile( file_path=seed_path, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index 7fc548ddaa..848edbf402 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -18,7 +18,12 @@ CompiledSource, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, + CompiledSourceView, +) from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, @@ -42,6 +47,7 @@ SourceEntry, ) from sqlbuild.tasks import task +from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes def fetch_events(_ctx: object) -> list[dict[str, object]]: @@ -501,14 +507,14 @@ def build_orders_project_graph() -> ProjectGraph: contents="sources: []\n", source_entries=(), ) - raw_orders_source: CompiledSource = CompiledSource( + raw_orders_source: CompiledSourceView = CompiledSource( key=raw_orders_key, deps=(), name="raw_orders", source_entry=SourceEntry(name="raw_orders", loader="raw_orders", managed=True), source_file=source_file, ) - orders_model: CompiledModel = CompiledModel( + orders_model: CompiledModelView = CompiledModel( key=orders_key, deps=(raw_orders_key,), name="orders", @@ -522,7 +528,7 @@ def build_orders_project_graph() -> ProjectGraph: qualified_name="orders", ), ) - project: CompiledProject = CompiledProject( + project: CompiledProjectView = CompiledProject( run_id="run-id", effective_target_name=None, effective_connection={}, @@ -530,7 +536,7 @@ def build_orders_project_graph() -> ProjectGraph: models=(orders_model,), sources=(raw_orders_source,), ) - return ProjectGraph( + return project_graph_from_indexes( project=project, upstream_deps={raw_orders_key: (), orders_key: (raw_orders_key,)}, downstream_deps={raw_orders_key: (orders_key,), orders_key: ()}, @@ -550,7 +556,7 @@ def build_model_depends_on_intermediate_loader_project_graph() -> ProjectGraph: resource_type=CompiledResourceType.SOURCE, name="fetch_pages", ) - return ProjectGraph( + return project_graph_from_indexes( project=project_graph.project, upstream_deps={ **project_graph.upstream_deps, diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/__init__.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py deleted file mode 100644 index 196c084397..0000000000 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/_test_types.py +++ /dev/null @@ -1,62 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ColumnEditTestCase: - description: str - sql: str - cascade: bool - expected_sql: str - expected_manual: str = "" - expected_passes_through: bool = False - - -@dataclass(frozen=True) -class HeaderEditTestCase: - description: str - contents: str - expected_contents: str - - -@dataclass(frozen=True) -class IdentifierSitesTestCase: - description: str - text: str - expected_offsets: tuple[int, ...] - - -@dataclass(frozen=True) -class ApplyEditsTestCase: - description: str - text: str - spans: tuple[tuple[int, int, str], ...] - expected_text: str - - -@dataclass(frozen=True) -class ApplyEditsErrorTestCase: - description: str - text: str - spans: tuple[tuple[int, int, str], ...] - expected_error: type[Exception] - - -@dataclass(frozen=True) -class CommitChangesTestCase: - description: str - files: dict[str, str] - current_files: dict[str, str] - locked_directory: str - spans: tuple[tuple[int, int, str], ...] - edited_path: str - moved_from: str - moved_to: str - expected_error: type[Exception] - expected_files: dict[str, str] - - -@dataclass(frozen=True) -class DeclarationEditTestCase: - description: str - contents: str - expected_contents: str diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py deleted file mode 100644 index cfe5233aff..0000000000 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/helpers.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Shared builders for refactoring helper tests.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.refactoring._helpers.columns.column_references import ( - analyze_column, - consumer_edits, -) -from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( - add_column_entry_edit, - column_config_edits, - column_entry_edits, - header_tokens, -) -from sqlbuild.compiler.refactoring._helpers.text.sql_sites import analysis_sql -from sqlbuild.compiler.refactoring._helpers.text.text_edits import text_edit -from sqlbuild.compiler.refactoring.models import ( - AnalysisSql, - BodyContext, - BodyEdits, - ColumnFacts, - ColumnQuery, - HeaderToken, - ProjectSqlFile, - TextEdit, -) -from sqlbuild.compiler.refactoring.types import EditKind, SqlFileRole - -DIALECT: str = "duckdb" -COLUMNS: dict[tuple[str, str], tuple[str, ...]] = { - ("ref", "fact_orders"): ("order_id", "customer_id", "amount"), - ("ref", "customers"): ("customer_id", "amount"), -} - - -def plan_consumer(*, sql: str, cascade: bool) -> BodyEdits: - """Plan renaming fact_orders.amount to revenue in one consumer query.""" - - analysis: AnalysisSql = analysis_sql(text=sql, dialect=DIALECT) - facts: ColumnFacts = analyze_column( - analysis=analysis, - dialect=DIALECT, - columns=COLUMNS, - query=ColumnQuery(column="amount", target_tables=analysis.tables[("ref", "fact_orders")]), - ) - return consumer_edits( - facts=facts, - context=BodyContext( - path="models/consumer.sql", - contents=sql, - map_span=lambda start, end: (start, end), - locate=lambda offset: offset, - fallback_offset=0, - ), - old="amount", - new="revenue", - cascade_root=cascade, - root_stars_pass=cascade, - ) - - -def migrated_column_edits(contents: str) -> tuple[TextEdit, ...]: - """Plan the header edits that rename amount to revenue with a column migration.""" - - tokens: tuple[HeaderToken, ...] = header_tokens(contents=contents) or () - added: tuple[TextEdit, ...] = tuple( - filter( - None, - (add_column_entry_edit(contents=contents, tokens=tokens, new="revenue", old="amount"),), - ) - ) - return ( - *column_config_edits(contents=contents, tokens=tokens, old="amount", new="revenue"), - *( - column_entry_edits( - contents=contents, tokens=tokens, old="amount", new="revenue", migrate=True - ) - or added - ), - ) - - -def span_edits(*, text: str, spans: tuple[tuple[int, int, str], ...]) -> tuple[TextEdit, ...]: - """Build column edits for (start, end, replacement) spans.""" - - return tuple( - text_edit(text=text, start=start, end=end, replacement=replacement, kind=EditKind.COLUMN) - for start, end, replacement in spans - ) - - -def write_tree(*, root: Path, files: dict[str, str]) -> None: - """Write project-relative files under a root.""" - - relative: str - contents: str - for relative, contents in files.items(): - _ = (root / relative).write_text(contents, encoding="utf-8") - - -def read_tree(*, root: Path) -> dict[str, str]: - """Read every file under a root, keyed by its relative path.""" - - return { - path.relative_to(root).as_posix(): path.read_text(encoding="utf-8") - for path in sorted(root.rglob("*.sql")) - } - - -def yaml_file(contents: str) -> ProjectSqlFile: - """Wrap YAML text as one project declaration file.""" - - return ProjectSqlFile( - relative_path="seeds/customers.yml", contents=contents, role=SqlFileRole.YAML - ) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py deleted file mode 100644 index 6baa3aebc1..0000000000 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_column_references.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Tests for column-rename edit planning inside one SQL body.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits -from sqlbuild.compiler.refactoring.models import BodyEdits -from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ColumnEditTestCase -from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import plan_consumer - - -@pytest.mark.parametrize( - "test_case", - [ - ColumnEditTestCase( - description="bare projection keeps its output name", - sql='SELECT o.amount FROM __ref("fact_orders") AS o', - cascade=False, - expected_sql='SELECT o.revenue AS amount FROM __ref("fact_orders") AS o', - ), - ColumnEditTestCase( - description="cascade renames a bare projection's output", - sql='SELECT o.amount FROM __ref("fact_orders") AS o', - cascade=True, - expected_sql='SELECT o.revenue FROM __ref("fact_orders") AS o', - expected_passes_through=True, - ), - ColumnEditTestCase( - description="aliased projection keeps its alias", - sql='SELECT o.amount AS total FROM __ref("fact_orders") AS o', - cascade=True, - expected_sql='SELECT o.revenue AS total FROM __ref("fact_orders") AS o', - ), - ColumnEditTestCase( - description="filter reference is renamed", - sql='SELECT order_id FROM __ref("fact_orders") WHERE amount > 0', - cascade=False, - expected_sql='SELECT order_id FROM __ref("fact_orders") WHERE revenue > 0', - ), - ColumnEditTestCase( - description="same-named column of another relation is untouched", - sql=( - 'SELECT c.amount FROM __ref("fact_orders") AS o ' - 'JOIN __ref("customers") AS c ON o.customer_id = c.customer_id' - ), - cascade=False, - expected_sql=( - 'SELECT c.amount FROM __ref("fact_orders") AS o ' - 'JOIN __ref("customers") AS c ON o.customer_id = c.customer_id' - ), - ), - ColumnEditTestCase( - description="star through a CTE is followed", - sql='WITH c AS (SELECT * FROM __ref("fact_orders")) SELECT c.amount FROM c', - cascade=False, - expected_sql=( - 'WITH c AS (SELECT * FROM __ref("fact_orders")) SELECT c.revenue AS amount FROM c' - ), - ), - ColumnEditTestCase( - description="root star needs cascade", - sql='SELECT * FROM __ref("fact_orders")', - cascade=False, - expected_sql='SELECT * FROM __ref("fact_orders")', - expected_manual="rerun with --cascade", - ), - ColumnEditTestCase( - description="root star passes through under cascade", - sql='SELECT * FROM __ref("fact_orders")', - cascade=True, - expected_sql='SELECT * FROM __ref("fact_orders")', - expected_passes_through=True, - ), - ColumnEditTestCase( - description="USING join is left for the author", - sql=( - 'SELECT o.order_id FROM __ref("fact_orders") AS o ' - 'JOIN __ref("customers") AS c USING (amount)' - ), - cascade=False, - expected_sql=( - 'SELECT o.order_id FROM __ref("fact_orders") AS o ' - 'JOIN __ref("customers") AS c USING (amount)' - ), - expected_manual="USING or NATURAL join", - ), - ], - ids=lambda case: case.description, -) -def test_given_consumer_sql_when_planning_column_rename_then_edits_match( - test_case: ColumnEditTestCase, -) -> None: - result: BodyEdits = plan_consumer(sql=test_case.sql, cascade=test_case.cascade) - - assert apply_text_edits(text=test_case.sql, edits=result.edits) == test_case.expected_sql - assert result.passes_through == test_case.expected_passes_through - reasons: tuple[str, ...] = tuple(location.reason for location in result.manual) - assert bool(reasons) == bool(test_case.expected_manual), reasons - assert all(test_case.expected_manual in reason for reason in reasons), reasons - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py deleted file mode 100644 index 5f922889e0..0000000000 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_declaration_edits.py +++ /dev/null @@ -1,232 +0,0 @@ -"""Tests for model and column reference rewriting in YAML and declaration headers.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( - consumer_column_header_edits, - model_name_header_edits, -) -from sqlbuild.compiler.refactoring._helpers.text.schema_edits import ( - schema_column_edits, - schema_model_name_edits, -) -from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits -from sqlbuild.compiler.refactoring._helpers.text.yaml_edits import ( - yaml_column_edits, - yaml_model_edits, -) -from sqlbuild.compiler.refactoring.models import TextEdit -from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( - DeclarationEditTestCase, -) -from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import yaml_file - -_BODY: str = "SELECT order_id FROM orders\n" - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationEditTestCase( - description="relationships target written as a reference", - contents=( - "seeds:\n - name: customers\n columns:\n - name: customer_id\n" - " audits:\n - relationships:\n" - ' to: __ref("fact_orders")\n field: customer_id\n' - ), - expected_contents=( - "seeds:\n - name: customers\n columns:\n - name: customer_id\n" - " audits:\n - relationships:\n" - ' to: __ref("order_facts")\n field: customer_id\n' - ), - ), - DeclarationEditTestCase( - description="bare relationships target in a flow mapping", - contents="audits:\n - relationships: {to: fact_orders, field: customer_id}\n", - expected_contents="audits:\n - relationships: {to: order_facts, field: customer_id}\n", - ), - DeclarationEditTestCase( - description="reference inside an escaped double-quoted expression", - contents='audits:\n - expression_is_true: {expression: "id IN (SELECT id FROM ' - '__ref(\\"fact_orders\\"))"}\n', - expected_contents='audits:\n - expression_is_true: {expression: "id IN (SELECT id ' - 'FROM __ref(\\"order_facts\\"))"}\n', - ), - DeclarationEditTestCase( - description="similar names and descriptions are left alone", - contents=( - "sources:\n - name: raw\n description: fact_orders feed\n" - " audits:\n - relationships: {to: fact_orders_daily, field: id}\n" - ), - expected_contents=( - "sources:\n - name: raw\n description: fact_orders feed\n" - " audits:\n - relationships: {to: fact_orders_daily, field: id}\n" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_yaml_when_renaming_model_then_references_follow( - test_case: DeclarationEditTestCase, -) -> None: - edits: tuple[tuple[str, TextEdit], ...] = yaml_model_edits( - files=(yaml_file(test_case.contents),), old="fact_orders", new="order_facts" - ) - - assert ( - apply_text_edits(text=test_case.contents, edits=tuple(edit for _, edit in edits)) - == test_case.expected_contents - ) - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationEditTestCase( - description="field of a relationship to the model", - contents="audits:\n - relationships: {to: '__ref(\"fact_orders\")', field: amount}\n", - expected_contents=( - "audits:\n - relationships: {to: '__ref(\"fact_orders\")', field: revenue}\n" - ), - ), - DeclarationEditTestCase( - description="field of a relationship to another model", - contents="audits:\n - relationships: {to: '__ref(\"customers\")', field: amount}\n", - expected_contents=( - "audits:\n - relationships: {to: '__ref(\"customers\")', field: amount}\n" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_yaml_when_renaming_column_then_relationship_fields_follow( - test_case: DeclarationEditTestCase, -) -> None: - edits: tuple[tuple[str, TextEdit], ...] = yaml_column_edits( - files=(yaml_file(test_case.contents),), model="fact_orders", old="amount", new="revenue" - ) - - assert ( - apply_text_edits(text=test_case.contents, edits=tuple(edit for _, edit in edits)) - == test_case.expected_contents - ) - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationEditTestCase( - description="MODEL header quoted SQL and bare relationships target", - contents=( - 'MODEL (\n audits [expression_is_true (expression "id IN (SELECT id FROM ' - "__ref('fact_orders'))\")],\n columns (\n id (audits [relationships " - f"(to fact_orders, field id)]),\n ),\n);\n{_BODY}" - ), - expected_contents=( - 'MODEL (\n audits [expression_is_true (expression "id IN (SELECT id FROM ' - "__ref('order_facts'))\")],\n columns (\n id (audits [relationships " - f"(to order_facts, field id)]),\n ),\n);\n{_BODY}" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_model_header_when_renaming_model_then_quoted_and_bare_references_follow( - test_case: DeclarationEditTestCase, -) -> None: - edits: tuple[TextEdit, ...] = model_name_header_edits( - contents=test_case.contents, old="fact_orders", new="order_facts" - ) - - assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationEditTestCase( - description="field of a referenced relationship target", - contents=( - "MODEL (\n columns (\n id (audits [relationships " - f'(to __ref("fact_orders"), field amount)]),\n ),\n);\n{_BODY}' - ), - expected_contents=( - "MODEL (\n columns (\n id (audits [relationships " - f'(to __ref("fact_orders"), field revenue)]),\n ),\n);\n{_BODY}' - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_model_header_when_renaming_column_then_relationship_fields_follow( - test_case: DeclarationEditTestCase, -) -> None: - edits: tuple[TextEdit, ...] = consumer_column_header_edits( - contents=test_case.contents, upstream="fact_orders", old="amount", new="revenue" - ) - - assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationEditTestCase( - description="bare target and quoted SQL in every SCHEMA of a file", - contents=( - "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " - "(to fact_orders, field id)]),\n ),\n);\n\n" - "SCHEMA (\n name order_totals,\n audits [expression_is_true (expression " - "\"total > (SELECT 0 FROM __ref('fact_orders') LIMIT 1)\")],\n);\n" - ), - expected_contents=( - "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " - "(to order_facts, field id)]),\n ),\n);\n\n" - "SCHEMA (\n name order_totals,\n audits [expression_is_true (expression " - "\"total > (SELECT 0 FROM __ref('order_facts') LIMIT 1)\")],\n);\n" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_schema_file_when_renaming_model_then_references_follow( - test_case: DeclarationEditTestCase, -) -> None: - edits: tuple[TextEdit, ...] = schema_model_name_edits( - contents=test_case.contents, old="fact_orders", new="order_facts" - ) - - assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationEditTestCase( - description="field of a relationship to the model", - contents=( - "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " - '(to __ref("fact_orders"), field amount)]),\n ),\n);\n' - ), - expected_contents=( - "SCHEMA (\n name order_shape,\n columns (\n id (audits [relationships " - '(to __ref("fact_orders"), field revenue)]),\n ),\n);\n' - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_schema_file_when_renaming_column_then_relationship_fields_follow( - test_case: DeclarationEditTestCase, -) -> None: - edits: tuple[TextEdit, ...] = schema_column_edits( - contents=test_case.contents, upstream="fact_orders", old="amount", new="revenue" - ) - - assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py deleted file mode 100644 index 78159601c5..0000000000 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_header_edits.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Tests for token-exact MODEL header edits.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.refactoring._helpers.text.header_edits import ( - insert_header_entry_edit, - model_name_header_edits, -) -from sqlbuild.compiler.refactoring._helpers.text.text_edits import apply_text_edits -from sqlbuild.compiler.refactoring.models import TextEdit -from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import HeaderEditTestCase -from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import migrated_column_edits - -_BODY: str = "SELECT order_id, amount FROM orders\n" - - -@pytest.mark.parametrize( - "test_case", - [ - HeaderEditTestCase( - description="declared column gains migrate_from before its metadata", - contents=f"MODEL (\n columns (\n amount (nullable false),\n ),\n);\n{_BODY}", - expected_contents=( - "MODEL (\n columns (\n revenue (migrate_from amount, nullable false),\n" - f" ),\n);\n{_BODY}" - ), - ), - HeaderEditTestCase( - description="bare declared column gains a migrate_from block", - contents=f"MODEL (\n columns (amount),\n);\n{_BODY}", - expected_contents=f"MODEL (\n columns (revenue (migrate_from amount)),\n);\n{_BODY}", - ), - HeaderEditTestCase( - description="undeclared column gets a new columns entry", - contents=f"MODEL (\n materialized incremental,\n);\n{_BODY}", - expected_contents=( - "MODEL (\n columns (revenue (migrate_from amount)),\n" - f" materialized incremental,\n);\n{_BODY}" - ), - ), - HeaderEditTestCase( - description="column-valued config follows the column", - contents=f"MODEL (\n unique_key [order_id, amount],\n);\n{_BODY}", - expected_contents=( - "MODEL (\n columns (revenue (migrate_from amount)),\n" - f" unique_key [order_id, revenue],\n);\n{_BODY}" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_header_when_renaming_migrated_column_then_header_is_rewritten( - test_case: HeaderEditTestCase, -) -> None: - edits: tuple[TextEdit, ...] = migrated_column_edits(test_case.contents) - - assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents - - -@pytest.mark.parametrize( - "test_case", - [ - HeaderEditTestCase( - description="cursor_inputs keys and relationships targets follow the model", - contents=( - "MODEL (\n cursor_inputs (\n fact_orders ordered_at,\n ),\n" - " columns (\n order_id (audits [relationships (to fact_orders, " - "field order_id)]),\n ),\n);\n" - f"{_BODY}" - ), - expected_contents=( - "MODEL (\n cursor_inputs (\n order_facts ordered_at,\n ),\n" - " columns (\n order_id (audits [relationships (to order_facts, " - "field order_id)]),\n ),\n);\n" - f"{_BODY}" - ), - ), - HeaderEditTestCase( - description="descriptions naming the model are left alone", - contents=f'MODEL (\n description "fact_orders summary",\n);\n{_BODY}', - expected_contents=f'MODEL (\n description "fact_orders summary",\n);\n{_BODY}', - ), - ], - ids=lambda case: case.description, -) -def test_given_header_when_renaming_model_then_references_follow( - test_case: HeaderEditTestCase, -) -> None: - edits: tuple[TextEdit, ...] = model_name_header_edits( - contents=test_case.contents, old="fact_orders", new="order_facts" - ) - - assert apply_text_edits(text=test_case.contents, edits=edits) == test_case.expected_contents - - -@pytest.mark.parametrize( - "test_case", - [ - HeaderEditTestCase( - description="multi-line header gets its own line", - contents=f"MODEL (\n materialized table,\n);\n{_BODY}", - expected_contents=( - f"MODEL (\n migrate_from fact_orders,\n materialized table,\n);\n{_BODY}" - ), - ), - HeaderEditTestCase( - description="empty header is expanded", - contents=f"MODEL ();\n{_BODY}", - expected_contents=f"MODEL (\n migrate_from fact_orders,\n);\n{_BODY}", - ), - ], - ids=lambda case: case.description, -) -def test_given_header_when_adding_model_migration_then_entry_is_inserted( - test_case: HeaderEditTestCase, -) -> None: - edit: TextEdit | None = insert_header_entry_edit( - contents=test_case.contents, - entry="migrate_from fact_orders", - display="migrate_from fact_orders", - ) - - assert edit is not None - assert apply_text_edits(text=test_case.contents, edits=(edit,)) == test_case.expected_contents - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py deleted file mode 100644 index d68fad0735..0000000000 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_text_edits.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Tests for authored-text edit application and identifier matching.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.refactoring._helpers.text.text_edits import ( - apply_text_edits, - identifier_sites, -) -from sqlbuild.compiler.refactoring.exceptions import RefactorEditError -from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( - ApplyEditsErrorTestCase, - ApplyEditsTestCase, - IdentifierSitesTestCase, -) -from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import span_edits - - -@pytest.mark.parametrize( - "test_case", - [ - ApplyEditsTestCase( - description="identical duplicate edits apply once", - text="SELECT amount FROM orders", - spans=((7, 13, "revenue"), (7, 13, "revenue")), - expected_text="SELECT revenue FROM orders", - ), - ApplyEditsTestCase( - description="insertions and replacements apply back to front", - text="SELECT amount FROM orders", - spans=((7, 13, "revenue"), (13, 13, " AS amount")), - expected_text="SELECT revenue AS amount FROM orders", - ), - ], - ids=lambda case: case.description, -) -def test_given_edits_when_applying_then_text_is_rewritten(test_case: ApplyEditsTestCase) -> None: - assert ( - apply_text_edits( - text=test_case.text, edits=span_edits(text=test_case.text, spans=test_case.spans) - ) - == test_case.expected_text - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ApplyEditsErrorTestCase( - description="overlapping replacements", - text="SELECT amount FROM orders", - spans=((7, 13, "revenue"), (10, 18, "x")), - expected_error=RefactorEditError, - ) - ], - ids=lambda case: case.description, -) -def test_given_overlapping_edits_when_applying_then_refuses( - test_case: ApplyEditsErrorTestCase, -) -> None: - with pytest.raises(test_case.expected_error): - _ = apply_text_edits( - text=test_case.text, edits=span_edits(text=test_case.text, spans=test_case.spans) - ) - - -@pytest.mark.parametrize( - "test_case", - [ - IdentifierSitesTestCase( - description="matches whole unquoted identifiers only", - text="__ref__orders AS (SELECT 1), __ref__orders_daily AS (SELECT 2)", - expected_offsets=(0,), - ), - IdentifierSitesTestCase( - description="skips comments and string literals", - text="-- __ref__orders\nSELECT '__ref__orders', __ref__orders", - expected_offsets=(41,), - ), - ], - ids=lambda case: case.description, -) -def test_given_text_when_finding_identifiers_then_code_sites_match( - test_case: IdentifierSitesTestCase, -) -> None: - sites: tuple[tuple[int, int, str], ...] = identifier_sites( - text=test_case.text, names=frozenset({"__ref__orders"}) - ) - - assert tuple(start for start, _, _ in sites) == test_case.expected_offsets - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py b/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py deleted file mode 100644 index c21491c1f0..0000000000 --- a/tests/unit/src/sqlbuild/compiler/refactoring/_helpers/test_workspace.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Tests for committing verified refactoring edits into a project.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.refactoring._helpers.project.workspace import commit_changes -from sqlbuild.compiler.refactoring.exceptions import RefactorWriteError -from sqlbuild.compiler.refactoring.models import FileChange -from tests.unit.src.sqlbuild.compiler.refactoring._helpers._test_types import ( - CommitChangesTestCase, -) -from tests.unit.src.sqlbuild.compiler.refactoring._helpers.helpers import ( - read_tree, - span_edits, - write_tree, -) - - -@pytest.mark.parametrize( - "test_case", - [ - CommitChangesTestCase( - description="a failed write restores every file", - files={"a.sql": "SELECT amount", "m.sql": "SELECT 1"}, - current_files={}, - locked_directory="locked", - spans=((7, 13, "revenue"),), - edited_path="a.sql", - moved_from="m.sql", - moved_to="locked/m.sql", - expected_error=PermissionError, - expected_files={"a.sql": "SELECT amount", "m.sql": "SELECT 1"}, - ), - CommitChangesTestCase( - description="a file edited since planning stops the write", - files={"a.sql": "SELECT amount", "m.sql": "SELECT 1"}, - current_files={"a.sql": "SELECT amount, 2"}, - locked_directory="open", - spans=((7, 13, "revenue"),), - edited_path="a.sql", - moved_from="m.sql", - moved_to="open/m.sql", - expected_error=RefactorWriteError, - expected_files={"a.sql": "SELECT amount, 2", "m.sql": "SELECT 1"}, - ), - ], - ids=lambda case: case.description, -) -def test_given_failing_commit_when_writing_then_project_is_unchanged( - tmp_path: Path, test_case: CommitChangesTestCase -) -> None: - write_tree(root=tmp_path, files={**test_case.files, **test_case.current_files}) - (tmp_path / test_case.locked_directory).mkdir() - (tmp_path / "locked").mkdir(exist_ok=True) - (tmp_path / "locked").chmod(0o500) - changes: tuple[FileChange, ...] = ( - FileChange( - path=test_case.edited_path, - original_path=test_case.edited_path, - edits=span_edits(text=test_case.files[test_case.edited_path], spans=test_case.spans), - ), - FileChange(path=test_case.moved_to, original_path=test_case.moved_from), - ) - - with pytest.raises(test_case.expected_error): - _ = commit_changes(project_dir=tmp_path, originals=test_case.files, changes=changes) - (tmp_path / "locked").chmod(0o700) - - assert read_tree(root=tmp_path) == test_case.expected_files - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py index feea050209..458ff145d2 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/_test_types.py @@ -5,7 +5,6 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_SCHEMA_VERSION from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity from sqlbuild.compiler.scopes.types import ( DeclarationKind, @@ -28,36 +27,32 @@ class PathNormalizationCase: @dataclass(frozen=True) -class PathNormalizationCacheCase: - """Expected bounded-cache behavior for repeated path normalization.""" +class QualifiedIdentityCase: + """One qualified identity round-trip case.""" description: str - path: str - call_count: int - expected_hits: int - expected_misses: int - expected_max_size: int - expected_current_size: int + text: str + expected_identity: ResourceIdentity | DeclarationIdentity @dataclass(frozen=True) -class PathVisibilityCase: - """One component-aware lexical visibility case.""" +class UnicodeKeyOrderCase: + """Resource names and paths, with the lookup mapping orders Python's `repr` sort gives.""" description: str - scope: ScopeKind - owner: str - resource: str - expected_visible: bool + names: tuple[str, ...] + expected_names: tuple[str, ...] + expected_paths: tuple[str, ...] @dataclass(frozen=True) -class QualifiedIdentityCase: - """One qualified identity round-trip case.""" +class IdentityReportCase: + """One qualified identity text, the identity a report names, and its diagnostic messages.""" description: str text: str - expected_identity: ResourceIdentity | DeclarationIdentity + expected_identity: str | None + expected_messages: tuple[str, ...] @dataclass(frozen=True) @@ -190,7 +185,7 @@ class CacheFaultCase: description: str mutate: Callable[..., None] - expected_schema_version: int = SCOPE_CACHE_SCHEMA_VERSION + expected_schema_version: int = 3 @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py index f9a33b4eca..e8263cb7de 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_identities.py @@ -4,48 +4,93 @@ import pytest -from sqlbuild.compiler.scopes._helpers.identities import format_identity, parse_identity from sqlbuild.compiler.scopes.exceptions import InvalidQualifiedIdentityError -from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind +from sqlbuild.compiler.scopes.main.build_scope_lookup import build_scope_lookup +from sqlbuild.compiler.scopes.main.explain_scope_declaration import explain_scope_declaration +from sqlbuild.compiler.scopes.main.query_scope_report import query_scope_report +from sqlbuild.compiler.scopes.models import ( + DeclarationExplanation, + DeclarationIdentity, + ScopeDiagnostic, + ScopeLookup, + ScopeReport, +) +from sqlbuild.compiler.scopes.types import DeclarationKind from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( + IdentityReportCase, InvalidIdentityCase, QualifiedIdentityCase, ) +from tests.unit.src.sqlbuild.compiler.scopes.helpers import scope_index + + +@pytest.mark.parametrize( + "test_case", + [IdentityReportCase("resource", "model:stg_orders", "model:stg_orders", ())], + ids=lambda case: case.description, +) +def test_given_qualified_resource_when_querying_then_report_names_the_same_identity( + test_case: IdentityReportCase, +) -> None: + lookup: ScopeLookup = build_scope_lookup(index=scope_index()) + + report: ScopeReport = query_scope_report(lookup=lookup, target=test_case.text) + + assert report.resource.identity == test_case.expected_identity + assert tuple(item.message for item in report.diagnostics) == test_case.expected_messages + + +@pytest.mark.parametrize( + "test_case", + [IdentityReportCase("public_declaration", "enum:order_status", "enum:order_status", ())], + ids=lambda case: case.description, +) +def test_given_qualified_declaration_when_explaining_then_identity_round_trips( + test_case: IdentityReportCase, +) -> None: + lookup: ScopeLookup = build_scope_lookup(index=scope_index()) + + explanation: DeclarationExplanation + diagnostics: tuple[ScopeDiagnostic, ...] + explanation, diagnostics = explain_scope_declaration(lookup=lookup, declaration=test_case.text) + + assert explanation.declaration is not None + assert explanation.declaration.identity == test_case.expected_identity + assert tuple(item.message for item in diagnostics) == test_case.expected_messages @pytest.mark.parametrize( "test_case", [ - QualifiedIdentityCase( - description="resource", - text="model:stg_orders", - expected_identity=ResourceIdentity(kind=ResourceKind.MODEL, name="stg_orders"), - ), - QualifiedIdentityCase( - description="public_declaration", - text="enum:order_status", - expected_identity=DeclarationIdentity(kind=DeclarationKind.ENUM, name="order_status"), - ), - QualifiedIdentityCase( - description="private_owner_qualified_declaration", - text="constant:model:stg_orders._minimum_value", - expected_identity=DeclarationIdentity( - kind=DeclarationKind.CONSTANT, - name="_minimum_value", - owner=ResourceIdentity(kind=ResourceKind.MODEL, name="stg_orders"), + IdentityReportCase( + "private_owner_qualified_declaration", + "constant:model:stg_orders._minimum_value", + None, + ( + "Target 'constant:model:stg_orders._minimum_value' is not valid for this " + "scope query", ), ), + IdentityReportCase( + "unknown_kind", + "widget:orders", + None, + ("Target 'widget:orders' is not valid for this scope query",), + ), ], ids=lambda case: case.description, ) -def test_given_qualified_identity_when_parsing_and_formatting_then_round_trips( - test_case: QualifiedIdentityCase, +def test_given_unknown_declaration_when_explaining_then_reports_its_formatted_identity( + test_case: IdentityReportCase, ) -> None: - parsed: ResourceIdentity | DeclarationIdentity = parse_identity(value=test_case.text) + lookup: ScopeLookup = build_scope_lookup(index=scope_index()) - assert parsed == test_case.expected_identity - assert format_identity(identity=parsed) == test_case.text + explanation: DeclarationExplanation + diagnostics: tuple[ScopeDiagnostic, ...] + explanation, diagnostics = explain_scope_declaration(lookup=lookup, declaration=test_case.text) + + assert explanation.declaration is test_case.expected_identity + assert tuple(item.message for item in diagnostics) == test_case.expected_messages @pytest.mark.parametrize( @@ -69,14 +114,18 @@ def test_given_cross_kind_same_name_when_constructing_identities_then_they_remai @pytest.mark.parametrize( "test_case", - [InvalidIdentityCase("bare_name", "stg_orders", InvalidQualifiedIdentityError)], + [InvalidIdentityCase("too_many_parts", "model:a:b:c", InvalidQualifiedIdentityError)], ids=lambda case: case.description, ) -def test_given_bare_name_when_parsing_then_rejects_unqualified_identity( +def test_given_unparseable_context_identity_when_explaining_then_rejects_unqualified_identity( test_case: InvalidIdentityCase, ) -> None: + lookup: ScopeLookup = build_scope_lookup(index=scope_index()) + with pytest.raises(test_case.expected_error, match="kind-qualified"): - parse_identity(value=test_case.value) + explain_scope_declaration( + lookup=lookup, declaration="enum:order_status", target=test_case.value + ) if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py index dc5617158a..19d1a90992 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_lookup_and_metadata.py @@ -15,6 +15,7 @@ DeclarationRecord, OwnershipRoot, ResourceIdentity, + ResourceRecord, ScopeIndex, ScopeLookup, ) @@ -27,6 +28,7 @@ DeclarationContainerProjectionCase, ExpectedBooleanCase, ExpectedErrorCase, + UnicodeKeyOrderCase, ) from tests.unit.src.sqlbuild.compiler.scopes.helpers import scope_index @@ -56,6 +58,56 @@ def test_given_unsorted_index_when_building_lookup_then_records_and_edges_are_ca ] +@pytest.mark.parametrize( + "test_case", + [ + UnicodeKeyOrderCase( + description="non_ascii_keys_follow_python_repr", + names=("caf\u00e9", "cafe", "\u00a0x", "\U0001f600", "Zed", "\u200bz", "e\u0301"), + expected_names=( + "Zed", + "\u200bz", + "\u00a0x", + "cafe", + "caf\u00e9", + "e\u0301", + "\U0001f600", + ), + expected_paths=( + "models/Zed.sql", + "models/\u200bz.sql", + "models/\u00a0x.sql", + "models/cafe.sql", + "models/caf\u00e9.sql", + "models/e\u0301.sql", + "models/\U0001f600.sql", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_non_ascii_keys_when_building_lookup_then_mappings_follow_python_repr_order( + test_case: UnicodeKeyOrderCase, +) -> None: + root: OwnershipRoot = OwnershipRoot("models", resource_kind=ResourceKind.MODEL) + resources: tuple[ResourceRecord, ...] = tuple( + ResourceRecord( + identity=ResourceIdentity(kind=ResourceKind.MODEL, name=name), + path=f"models/{name}.sql", + ownership_root=root, + ) + for name in test_case.names + ) + + lookup: ScopeLookup = build_scope_lookup( + index=ScopeIndex(ownership_roots=(root,), resources=resources) + ) + + assert tuple(identity.name for identity in lookup.resources) == test_case.expected_names + assert tuple(lookup.resources_by_path) == test_case.expected_paths + assert tuple(sorted(lookup.resources, key=repr)) == tuple(lookup.resources) + + @pytest.mark.parametrize( "test_case", [ExpectedErrorCase("mapping_proxy", TypeError)], diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py index f0bf863cd9..27e4b22cb8 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_offline_cache.py @@ -223,10 +223,10 @@ def test_given_project_mutation_when_fingerprinting_then_only_scope_inputs_inval path: Path = tmp_path / test_case.path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(test_case.initial, encoding="utf-8") - before: str = scope_index_fingerprint(project_dir=tmp_path) + before: str | None = scope_index_fingerprint(project_dir=tmp_path) path.write_text(test_case.updated, encoding="utf-8") - after: str = scope_index_fingerprint(project_dir=tmp_path) + after: str | None = scope_index_fingerprint(project_dir=tmp_path) assert (before != after) is test_case.expected_changes_fingerprint @@ -264,15 +264,15 @@ def test_given_source_add_rename_and_delete_when_fingerprinting_then_paths_inval write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: write_scope_cache_project(root=tmp_path, write_repo_files=write_repo_files) - original: str = scope_index_fingerprint(project_dir=tmp_path) + original: str | None = scope_index_fingerprint(project_dir=tmp_path) added_path: Path = tmp_path / "models" / "added.sql" added_path.write_text(_MODEL, encoding="utf-8") - added: str = scope_index_fingerprint(project_dir=tmp_path) + added: str | None = scope_index_fingerprint(project_dir=tmp_path) renamed_path: Path = tmp_path / "models" / "renamed.sql" added_path.rename(renamed_path) - renamed: str = scope_index_fingerprint(project_dir=tmp_path) + renamed: str | None = scope_index_fingerprint(project_dir=tmp_path) renamed_path.unlink() - restored: str = scope_index_fingerprint(project_dir=tmp_path) + restored: str | None = scope_index_fingerprint(project_dir=tmp_path) result: bool = len({original, added, renamed}) == 3 and restored == original assert result is test_case.expected_result diff --git a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py index a16976601d..7abb2d4126 100644 --- a/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py +++ b/tests/unit/src/sqlbuild/compiler/scopes/test_paths.py @@ -1,16 +1,12 @@ -"""Tests for pure normalized path visibility.""" +"""Tests for project-relative scope path normalization.""" from __future__ import annotations import pytest -from sqlbuild.compiler.scopes._helpers.paths import is_equal_or_descendant, normalize_path -from sqlbuild.compiler.scopes.types import ScopeKind -from tests.unit.src.sqlbuild.compiler.scopes._test_types import ( - PathNormalizationCacheCase, - PathNormalizationCase, - PathVisibilityCase, -) +from sqlbuild.compiler.scopes._helpers.paths import normalize_path +from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError +from tests.unit.src.sqlbuild.compiler.scopes._test_types import PathNormalizationCase @pytest.mark.parametrize( @@ -43,49 +39,17 @@ def test_given_posix_or_windows_path_when_normalizing_then_returns_canonical_pos @pytest.mark.parametrize( "test_case", [ - PathNormalizationCacheCase( - description="repeated canonical path uses one normalization result", - path="models/commerce/orders.sql", - call_count=1_000, - expected_hits=999, - expected_misses=1, - expected_max_size=65_536, - expected_current_size=1, - ) + PathNormalizationCase("absolute", "/models/orders.sql", "project-relative"), + PathNormalizationCase("windows_drive", "C:\\models\\orders.sql", "project-relative"), + PathNormalizationCase("escaping", "models/../../orders.sql", "escapes the project"), ], ids=lambda case: case.description, ) -def test_given_repeated_path_when_normalizing_then_reuses_bounded_cache( - test_case: PathNormalizationCacheCase, -) -> None: - normalize_path.cache_clear() - - for _ in range(test_case.call_count): - _ = normalize_path(path=test_case.path) - - assert normalize_path.cache_info().hits == test_case.expected_hits - assert normalize_path.cache_info().misses == test_case.expected_misses - assert normalize_path.cache_info().maxsize == test_case.expected_max_size - assert normalize_path.cache_info().currsize == test_case.expected_current_size - normalize_path.cache_clear() - - -@pytest.mark.parametrize( - "test_case", - [ - PathVisibilityCase( - "component_prefix", ScopeKind.INHERITED, "models/order", "models/orders", False - ) - ], - ids=lambda case: case.description, -) -def test_given_component_prefix_only_when_comparing_descendant_then_returns_false( - test_case: PathVisibilityCase, +def test_given_path_outside_project_when_normalizing_then_rejects_it( + test_case: PathNormalizationCase, ) -> None: - assert ( - is_equal_or_descendant(path=test_case.resource, ancestor=test_case.owner) - is test_case.expected_visible - ) + with pytest.raises(InvalidScopePathError, match=test_case.expected_path): + normalize_path(path=test_case.path) if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index d0c43013f9..cbca5cf59e 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -12,7 +12,11 @@ CompiledRelationLocation, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.planner.models import FunctionPlanEntry, ModelPlanEntry from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention @@ -154,7 +158,7 @@ def build_clone_function_entry( ) -def build_clone_retention_project() -> CompiledProject: +def build_clone_retention_project() -> CompiledProjectView: return CompiledProject( run_id="clone-retention-run", effective_target_name="test", @@ -167,7 +171,7 @@ def build_clone_retention_project() -> CompiledProject: ) -def _build_retention_model(*, name: str, schema: str) -> CompiledModel: +def _build_retention_model(*, name: str, schema: str) -> CompiledModelView: key: CompiledObjectKey = CompiledObjectKey( resource_type=CompiledResourceType.MODEL, name=name, diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index ae6494a16c..05cb83d893 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -29,7 +29,11 @@ CompiledSource, CompileModelConfig, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledProjectView, + CompiledResourceType, + CompiledSourceView, +) from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME @@ -541,8 +545,8 @@ def build_project( destination_database: str | None = None, destination_schema: str = "analytics", migrate_from: str | None = None, -) -> CompiledProject: - source: CompiledSource = CompiledSource( +) -> CompiledProjectView: + source: CompiledSourceView = CompiledSource( key=CompiledObjectKey( resource_type=CompiledResourceType.SOURCE, name="raw_orders", @@ -557,7 +561,7 @@ def build_project( ), source_file=cast(Any, object()), ) - sources: tuple[CompiledSource, ...] = {True: (), False: (source,)}[source_schema is None] + sources: tuple[CompiledSourceView, ...] = {True: (), False: (source,)}[source_schema is None] return CompiledProject( run_id="run-1", effective_target_name="dev", diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index dcd6964ca9..490313af2a 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -11,13 +11,12 @@ from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, CompiledRelationLocation, CompileModelConfig, CompileModelInput, CompileProjectInputs, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSqlModelFile, @@ -397,7 +396,7 @@ def build_manifest_seed_node( return node -def build_compiled_project_with_models(sql_by_model_name: dict[str, str]) -> CompiledProject: +def build_compiled_project_with_models(sql_by_model_name: dict[str, str]) -> CompiledProjectView: """Build a minimal compiled project from model SQL strings.""" return build_compiled_project_with_model_specs( @@ -412,7 +411,7 @@ def build_compiled_project_with_model_specs( sql_by_model_name: dict[str, str], tags_by_model_name: dict[str, tuple[str, ...]], path_by_model_name: dict[str, str], -) -> CompiledProject: +) -> CompiledProjectView: """Build a minimal compiled project from model SQL, tags, and relative paths.""" model_inputs: list[CompileModelInput] = [] diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py index e345460214..936f1bdaff 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_graph.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.graph.core import ( build_dbt_combined_graph, expand_combined_downstream, @@ -118,7 +118,7 @@ def test_given_manifest_and_compiled_project_when_building_graph_then_returns_ex test_case: DbtCombinedGraphTestCase, ) -> None: manifest: DbtManifestIndex = build_dbt_manifest_index(raw_data=test_case.manifest_data) - project: CompiledProject = build_compiled_project_with_models( + project: CompiledProjectView = build_compiled_project_with_models( test_case.sqlbuild_model_sql_by_name ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py index f0699c4994..77e56f82b8 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_plan_orchestration.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index @@ -720,7 +720,7 @@ def test_given_dbt_interop_inputs_when_planning_then_returns_expected_plan( test_case: DbtPlanOrchestrationTestCase, ) -> None: manifest: DbtManifestIndex = build_dbt_manifest_index(raw_data=test_case.manifest_data) - project: CompiledProject = build_compiled_project_with_model_specs( + project: CompiledProjectView = build_compiled_project_with_model_specs( sql_by_model_name=test_case.sqlbuild_model_sql_by_name, tags_by_model_name=test_case.sqlbuild_model_tags_by_name, path_by_model_name=test_case.sqlbuild_model_path_by_name, @@ -798,7 +798,7 @@ def test_given_failing_dbt_ls_when_planning_then_raises_coded_error( test_case: DbtPlanOrchestrationErrorTestCase, ) -> None: manifest: DbtManifestIndex = build_dbt_manifest_index(raw_data=_MANIFEST_DATA) - project: CompiledProject = build_compiled_project_with_model_specs( + project: CompiledProjectView = build_compiled_project_with_model_specs( sql_by_model_name=_SQLBUILD_SQL, tags_by_model_name=_SQLBUILD_TAGS, path_by_model_name=_SQLBUILD_PATHS, diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py index 2ca84382e4..2add74343b 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/test_selection.py @@ -2,7 +2,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.integrations.dbt._helpers.graph.core import build_dbt_combined_graph from sqlbuild.integrations.dbt._helpers.manifest.core import build_dbt_manifest_index from sqlbuild.integrations.dbt._helpers.selection.core import resolve_dbt_interop_sqlbuild_selection @@ -480,7 +480,7 @@ def test_given_dbt_interop_selectors_when_resolving_then_returns_expected_sqlbui test_case: DbtSelectionTestCase, ) -> None: manifest: DbtManifestIndex = build_dbt_manifest_index(raw_data=test_case.manifest_data) - project: CompiledProject = build_compiled_project_with_model_specs( + project: CompiledProjectView = build_compiled_project_with_model_specs( sql_by_model_name=test_case.sqlbuild_model_sql_by_name, tags_by_model_name=test_case.sqlbuild_model_tags_by_name, path_by_model_name=test_case.sqlbuild_model_path_by_name, @@ -539,7 +539,7 @@ def test_given_invalid_dbt_interop_path_selector_when_resolving_then_raises_clea test_case: DbtSelectionErrorTestCase, ) -> None: manifest: DbtManifestIndex = build_dbt_manifest_index(raw_data=test_case.manifest_data) - project: CompiledProject = build_compiled_project_with_model_specs( + project: CompiledProjectView = build_compiled_project_with_model_specs( sql_by_model_name=test_case.sqlbuild_model_sql_by_name, tags_by_model_name=test_case.sqlbuild_model_tags_by_name, path_by_model_name=test_case.sqlbuild_model_path_by_name, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py index 75209e1cf2..61b2d9ab41 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py @@ -213,23 +213,6 @@ class NativeBuildIdentityTestCase: expected_warm_evaluations: int -@dataclass(frozen=True) -class NativeRequestBuildErrorTestCase: - """One programming error raised while building the native rules request.""" - - description: str - expected_error: Exception - - -@dataclass(frozen=True) -class NativeRequestEncodeErrorTestCase: - """One request value the JSON encoder rejects.""" - - description: str - rejected_value: object - expected_message: str - - @dataclass(frozen=True) class IncrementalRulesModeTestCase: """Evaluation options under which incremental results must equal full evaluation.""" diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 1bc91d3624..e4378bf8b7 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -1,8 +1,7 @@ """Test helpers for native rules engine boundaries.""" import json -from collections.abc import Callable -from dataclasses import dataclass, field, replace +from dataclasses import dataclass, field from operator import attrgetter from pathlib import Path from typing import Any, cast @@ -13,22 +12,24 @@ CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, CompiledObjectKey, - CompiledProject, CompiledSqlTest, CompiledSqlTestResource, CompileSqlTestCte, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine import native from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached +from sqlbuild.rule_engine._helpers.run import native_rows from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import ( CustomRulesOutcome, Finding, + NativeRulesEvaluation, Rule, RuleExemption, RuleIgnore, @@ -42,46 +43,48 @@ _MODEL_SQL: str = "WITH final AS (SELECT 1 AS id) SELECT id FROM final" -def project_with_scope(*, index: ScopeIndex) -> CompiledProject: +def project_with_scope(*, index: ScopeIndex) -> CompiledProjectView: """Return a minimal compiled project carrying canonical scope facts.""" - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="orders", relative_path="models/orders.sql", sql="SELECT 1 AS id", config_values={}, ) - return replace(project, scope_index=index) + return project_with(project=project, scope_index=index) def captured_native_request( - *, monkeypatch: pytest.MonkeyPatch, project: CompiledProject, project_dir: Path + *, monkeypatch: pytest.MonkeyPatch, project: CompiledProjectView, project_dir: Path ) -> dict[str, Any]: - """Evaluate through the Python boundary and capture the serialized request.""" + """Evaluate through the Python boundary and capture the header and rows handed to Rust.""" captured: dict[str, Any] = {} - def parse_rules_parts( - request_json: bytes, model_jsons: list[bytes], model_digests: list[str] - ) -> object: - captured.update(cast(dict[str, Any], json.loads(request_json))) - captured["models"] = [json.loads(model_json) for model_json in model_jsons] - return object() - - def evaluate_parsed_rules(parsed: object) -> str: - return json.dumps( - { - "version": 1, - "faults": [], - "evaluated_models": 0, - "cache_hits": 0, - "cache_misses": 0, - "selected_codes": [], - } + def build_rules_request( + project_rows: dict[str, Any], + models: list[tuple[object, ...]], + sql_tests: list[tuple[object, ...]], + sql_scenarios: list[tuple[object, ...]], + ) -> tuple[object, tuple[int, int, int]]: + captured.update(cast(dict[str, Any], json.loads(project_rows["header_json"]))) + captured.update(models=models, sql_tests=sql_tests, sql_scenarios=sql_scenarios) + return object(), (len(models), len(sql_tests), len(sql_scenarios)) + + def evaluate_rules_request(request: object) -> NativeRulesEvaluation: + return NativeRulesEvaluation( + findings=(), + selected_codes=(), + evaluated_models=0, + cache_hits=0, + cache_misses=0, + built_in_ms=0, + reused=False, ) - monkeypatch.setattr(native._native, "parse_rules_parts", parse_rules_parts) - monkeypatch.setattr(native._native, "evaluate_parsed_rules", evaluate_parsed_rules) + monkeypatch.setattr(native_rows._native, "build_rules_request", build_rules_request) + monkeypatch.setattr(native, "evaluate_rules_request", evaluate_rules_request) native.evaluate_native( project=project, config=RulesConfig(cache=RulesCacheConfig(enabled=False)), @@ -220,7 +223,7 @@ def load_custom_rule(*, root: Path, configured_path: Path) -> Rule: def custom_rule_inputs( *, tmp_path: Path, test_case: CustomRuleTestCase -) -> tuple[CompiledProject, RulesConfig]: +) -> tuple[CompiledProjectView, RulesConfig]: _ = write_rule( root=tmp_path, body=test_case.body, @@ -228,7 +231,7 @@ def custom_rule_inputs( project_wide=test_case.project_wide, ) (tmp_path / "rules" / "input.json").write_text("{}", encoding="utf-8") - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="commerce__mart__orders", relative_path="models/mart/commerce__mart__orders.sql", sql=_MODEL_SQL, @@ -262,26 +265,26 @@ def custom_rules_with_imports( def two_model_project( *, customers_sql: str, customers_config: dict[str, object] -) -> CompiledProject: +) -> CompiledProjectView: """Return an orders and customers project with configurable customers facts.""" - orders: CompiledProject = build_project( + orders: CompiledProjectView = build_project( name="orders", relative_path="models/orders.sql", sql="SELECT 1 AS order_id", config_values={}, ) - customers: CompiledProject = build_project( + customers: CompiledProjectView = build_project( name="customers", relative_path="models/customers.sql", sql=customers_sql, config_values=customers_config, ) - return replace(orders, models=(*orders.models, *customers.models)) + return project_with(project=orders, models=(*orders.models, *customers.models)) def evaluate_cached_custom_rules( - *, project: CompiledProject, project_dir: Path, cache_enabled: bool + *, project: CompiledProjectView, project_dir: Path, cache_enabled: bool ) -> CustomRulesOutcome: """Evaluate the selected custom rule through the incremental read-tracking cache.""" @@ -304,7 +307,7 @@ def evaluate_contract_rule( config: RulesConfig = RulesConfig( select=("SQBRCONTRACT101",), cache=RulesCacheConfig(enabled=cache_enabled) ) - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="commerce__mart__orders", relative_path="models/mart/commerce__mart__orders.sql", sql="WITH orders AS (SELECT id FROM source_orders) SELECT id FROM orders", @@ -313,20 +316,6 @@ def evaluate_contract_rule( return evaluate(project=project, config=config, project_dir=project_dir) -def record_native_evaluations(*, monkeypatch: pytest.MonkeyPatch) -> list[object]: - """Count every request that reaches the native built-in rules engine.""" - - requests: list[object] = [] - evaluate_parsed_rules: Callable[[Any], str] = native._native.evaluate_parsed_rules - - def recording(parsed: Any) -> str: - requests.append(parsed) - return evaluate_parsed_rules(parsed) - - monkeypatch.setattr(native._native, "evaluate_parsed_rules", recording) - return requests - - @dataclass(frozen=True) class IncrementalModelSpec: """One synthetic compiled model in an incremental built-in rules step.""" @@ -394,8 +383,8 @@ def evaluate_incremental_step( ) -def _incremental_project(models: tuple[IncrementalModelSpec, ...]) -> CompiledProject: - projects: list[CompiledProject] = [ +def _incremental_project(models: tuple[IncrementalModelSpec, ...]) -> CompiledProjectView: + projects: list[CompiledProjectView] = [ build_project( name=model.name, relative_path=model.relative_path, @@ -404,4 +393,6 @@ def _incremental_project(models: tuple[IncrementalModelSpec, ...]) -> CompiledPr ) for model in models ] - return replace(projects[0], models=tuple(project.models[0] for project in projects)) + return project_with( + project=projects[0], models=tuple(project.models[0] for project in projects) + ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py index 795eb9048f..bdded55451 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_evidence.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( _RuleTestEvidence, @@ -288,7 +288,7 @@ def test_given_collectable_rule_evidence_when_evaluating_then_coverage_fault_cle tmp_path: Path, test_case: CustomRuleEvidenceTestCase, ) -> None: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs( tmp_path=tmp_path, @@ -428,7 +428,7 @@ def test_given_cached_coverage_fault_when_evidence_changes_then_cache_is_invalid tmp_path: Path, test_case: CustomRuleEvidenceTestCase, ) -> None: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs( tmp_path=tmp_path, diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py index 617e9cf8b9..6906304ca6 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py @@ -7,7 +7,7 @@ import pytest import sqlbuild._native as native_module -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine import custom_rules from sqlbuild.rule_engine.models import CustomRulesOutcome from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( @@ -94,8 +94,10 @@ def test_given_recorded_reads_when_one_model_changes_then_only_dependent_results test_case: CustomRuleReadTrackingTestCase, ) -> None: _ = write_rule(root=tmp_path, body=test_case.body, module_import=test_case.module_prelude) - original: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) - edited: CompiledProject = two_model_project( + original: CompiledProjectView = two_model_project( + customers_sql=_CUSTOMERS_SQL, customers_config={} + ) + edited: CompiledProjectView = two_model_project( customers_sql=test_case.edited_customers_sql, customers_config=test_case.edited_customers_config, ) @@ -146,7 +148,9 @@ def test_given_private_view_access_when_evaluating_repeatedly_then_rule_is_never test_case: CustomRuleUncacheableTestCase, ) -> None: _ = write_rule(root=tmp_path, body=test_case.body) - project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + project: CompiledProjectView = two_model_project( + customers_sql=_CUSTOMERS_SQL, customers_config={} + ) cold: CustomRulesOutcome = evaluate_cached_custom_rules( project=project, project_dir=tmp_path, cache_enabled=True @@ -193,7 +197,9 @@ def test_given_untracked_rule_reading_hidden_file_when_file_changes_then_rule_re body=test_case.body, module_import=test_case.module_prelude, ) - project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + project: CompiledProjectView = two_model_project( + customers_sql=_CUSTOMERS_SQL, customers_config={} + ) cold: CustomRulesOutcome = evaluate_cached_custom_rules( project=project, project_dir=tmp_path, cache_enabled=True @@ -244,7 +250,9 @@ def test_given_file_edited_while_rules_run_when_rerunning_then_new_contents_are_ body=test_case.body, module_import=test_case.module_prelude, ) - project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + project: CompiledProjectView = two_model_project( + customers_sql=_CUSTOMERS_SQL, customers_config={} + ) run_host: Callable[..., object] = custom_rules.run_custom_hosts def run_host_then_edit(**kwargs: Any) -> object: @@ -286,7 +294,9 @@ def test_given_custom_rule_cache_from_another_native_build_when_evaluating_then_ test_case: NativeBuildIdentityTestCase, ) -> None: _ = write_rule(root=tmp_path, body=_FLAG_ORDERS.format(condition="True").strip()) - project: CompiledProject = two_model_project(customers_sql=_CUSTOMERS_SQL, customers_config={}) + project: CompiledProjectView = two_model_project( + customers_sql=_CUSTOMERS_SQL, customers_config={} + ) cold: CustomRulesOutcome = evaluate_cached_custom_rules( project=project, project_dir=tmp_path, cache_enabled=True diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 12f1d9a917..9c013f18a7 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -5,7 +5,10 @@ import pytest -from sqlbuild.compiler.compile.models import CompactLineageFacts, CompiledProject +from sqlbuild.compiler.compile.models import CompactLineageFacts +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import ( @@ -60,7 +63,7 @@ def test_given_selected_custom_rule_when_evaluating_then_reports_fault( tmp_path: Path, test_case: CustomRuleTestCase, ) -> None: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs(tmp_path=tmp_path, test_case=test_case) @@ -85,7 +88,7 @@ def test_given_custom_rule_when_repeated_then_deterministic_cache_is_reused( tmp_path: Path, test_case: CustomRuleTestCase, ) -> None: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs(tmp_path=tmp_path, test_case=test_case) _ = evaluate(project=project, config=config, project_dir=tmp_path) @@ -111,7 +114,7 @@ def test_given_lazy_lineage_when_materialized_then_project_rule_cache_is_stable( tmp_path: Path, test_case: CustomRuleTestCase, ) -> None: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs(tmp_path=tmp_path, test_case=test_case) lineage: CompactLineageFacts = CompactLineageFacts( @@ -119,14 +122,16 @@ def test_given_lazy_lineage_when_materialized_then_project_rule_cache_is_stable( rows=((0, 0, 1, ()),), resource_name_indexes={0: 0}, ) - project = replace( - project, - models=(replace(project.models[0], fast_lineage_columns=lineage),), + project = project_with( + project=project, + models=(replace(compiled_model_record(project.models[0]), fast_lineage_columns=lineage),), ) _ = evaluate(project=project, config=config, project_dir=tmp_path) - materialized_project: CompiledProject = replace( - project, - models=(replace(project.models[0], fast_lineage_columns=tuple(lineage)),), + materialized_project: CompiledProjectView = project_with( + project=project, + models=( + replace(compiled_model_record(project.models[0]), fast_lineage_columns=tuple(lineage)), + ), ) second: RulesResult = evaluate( @@ -153,7 +158,7 @@ def test_given_hermetic_custom_rule_when_cache_required_then_second_run_hits_cac tmp_path: Path, test_case: CustomRuleTestCase, ) -> None: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs(tmp_path=tmp_path, test_case=test_case) _ = evaluate(project=project, config=config, project_dir=tmp_path) @@ -197,7 +202,7 @@ def test_given_project_file_change_when_repeating_custom_rule_then_dependencies_ tmp_path: Path, test_case: CustomRuleCacheDependencyTestCase, ) -> None: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs( tmp_path=tmp_path, @@ -247,7 +252,7 @@ def test_given_nonhermetic_custom_rule_when_cache_required_then_raises_file_line tmp_path: Path, test_case: CustomRuleTestCase, ) -> None: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs(tmp_path=tmp_path, test_case=test_case) with pytest.raises(RulesError, match=test_case.expected_error_pattern): @@ -270,7 +275,7 @@ def test_given_project_wide_rule_when_reading_project_context_then_cache_is_allo tmp_path: Path, test_case: CustomRuleTestCase, ) -> None: - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs(tmp_path=tmp_path, test_case=test_case) @@ -318,7 +323,7 @@ def test_given_common_suppression_when_custom_rule_faults_then_native_policy_sup description=test_case.description, body="return [ctx.finding(subject=model)]", ) - project: CompiledProject + project: CompiledProjectView config: RulesConfig project, config = custom_rule_inputs(tmp_path=tmp_path, test_case=custom_rule_case) relative_path: str = "models/mart/commerce__mart__orders.sql" diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index 586c0a081f..80fe76fc77 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -1,4 +1,3 @@ -from dataclasses import replace from pathlib import Path from typing import Any @@ -6,11 +5,11 @@ from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledProject, CompiledSqlScenario, CompiledSqlTest, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredSqlScenarioFile, ) @@ -41,7 +40,7 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar tmp_path: Path, test_case: NativeFactPayloadTestCase, ) -> None: - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="orders", relative_path="models/orders.sql", sql="SELECT 1", @@ -80,61 +79,53 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar request: dict[str, Any] = captured_native_request( monkeypatch=monkeypatch, - project=replace(project, sql_tests=sql_tests, sql_scenarios=(scenario,)), + project=project_with(project=project, sql_tests=sql_tests, sql_scenarios=(scenario,)), project_dir=tmp_path, ) assert len(request["sql_tests"]) == test_case.expected_test_count assert request["sql_tests"][:3] == [ - { - "source_path": f"tests/unit/{name}.sql", - "ownership_root": "tests/unit", - "block_index": index, - "name": name, - "explicit_name": name, - "mode": mode, - "expected_model_names": [], - "assertion_names": [], - "assertion_target_model_names": [], - "target_model_names": [], - "tested_resources": [{"kind": mode, "name": name}], - } + ( + f"tests/unit/{name}.sql", + "tests/unit", + index, + name, + name, + mode, + ([], [], [], []), + [(mode, name)], + None, + None, + ) for index, (mode, name) in enumerate( (("macro", "normalize"), ("udf", "tax"), ("table_fn", "items")), start=1 ) ] - assert request["sql_tests"][3] == { - "source_path": "tests/unit/test_orders__keeps_paid.sql", - "ownership_root": "tests/unit", - "block_index": 1, - "name": "orders: keeps paid orders", - "explicit_name": "orders: keeps paid orders", - "mode": "model", - "expected_model_names": ["orders"], - "assertion_names": ["paid"], - "assertion_target_model_names": ["orders"], - "target_model_names": ["orders"], - "tested_resources": [], - "authored_ctes": [], - "expected_ctes": [], - "assertion_ctes": [], - "has_macro_mocks": False, - "has_model_query_overrides": False, - } + assert request["sql_tests"][3] == ( + "tests/unit/test_orders__keeps_paid.sql", + "tests/unit", + 1, + "orders: keeps paid orders", + "orders: keeps paid orders", + "model", + (["orders"], ["paid"], ["orders"], ["orders"]), + [], + ([], [], [], False, False), + None, + ) assert len(request["sql_scenarios"]) == test_case.expected_scenario_count assert request["sql_scenarios"] == [ - { - "source_path": "tests/scenarios/orders.sql", - "ownership_root": "tests/scenarios", - "name": "orders", - "description": "orders remain valid", - "expected_model_names": ["orders"], - "assertion_names": ["positive"], - "assertion_target_model_names": ["payments"], - "target_model_names": ["orders", "payments"], - } + ( + "tests/scenarios/orders.sql", + "tests/scenarios", + "orders", + "orders remain valid", + ["orders"], + ["positive"], + ["payments"], + ["orders", "payments"], + ) ] - assert request["models"][0]["targeting_test_count"] == 1 assert "/private/project" not in str(request["sql_tests"] + request["sql_scenarios"]) assert "secret fixture value" not in str(request["sql_tests"] + request["sql_scenarios"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py index 3273ff01c4..e66b18c738 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_memo.py @@ -2,16 +2,9 @@ import pytest -import sqlbuild._native as native_module from sqlbuild.rule_engine.models import RulesResult -from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( - NativeBuildIdentityTestCase, - NativeMemoTestCase, -) -from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( - evaluate_contract_rule, - record_native_evaluations, -) +from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import NativeMemoTestCase +from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule @pytest.mark.parametrize( @@ -61,34 +54,5 @@ def test_given_warm_built_in_memo_when_request_repeats_or_changes_then_matches_u assert edited.cache_misses > 0 -@pytest.mark.parametrize( - "test_case", - [NativeBuildIdentityTestCase("memoized response follows the native build", 1, 0)], - ids=lambda case: case.description, -) -def test_given_memoized_response_from_another_native_build_when_evaluating_then_rules_rerun( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - test_case: NativeBuildIdentityTestCase, -) -> None: - config_values: dict[str, object] = {"materialized": "table"} - cold: RulesResult = evaluate_contract_rule( - config_values=config_values, project_dir=tmp_path, cache_enabled=True - ) - native_calls: list[object] = record_native_evaluations(monkeypatch=monkeypatch) - monkeypatch.setattr(native_module, "BUILD_IDENTITY", f"{native_module.BUILD_IDENTITY}-rebuilt") - rebuilt: RulesResult = evaluate_contract_rule( - config_values=config_values, project_dir=tmp_path, cache_enabled=True - ) - rebuilt_calls: int = len(native_calls) - warm: RulesResult = evaluate_contract_rule( - config_values=config_values, project_dir=tmp_path, cache_enabled=True - ) - - assert rebuilt_calls == test_case.expected_rebuilt_evaluations - assert len(native_calls) - rebuilt_calls == test_case.expected_warm_evaluations - assert rebuilt.findings == warm.findings == cold.findings - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_request.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_request.py deleted file mode 100644 index 0715875638..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_request.py +++ /dev/null @@ -1,150 +0,0 @@ -from pathlib import Path -from typing import NoReturn - -import pytest - -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.rule_engine._helpers.engine import native -from sqlbuild.rule_engine.models import RulesCacheConfig, RulesConfig -from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( - NativeRequestBuildErrorTestCase, - NativeRequestEncodeErrorTestCase, -) -from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project - - -@pytest.mark.parametrize( - "test_case", - [ - NativeRequestBuildErrorTestCase( - description="type error", expected_error=TypeError("orders payload is not a mapping") - ), - NativeRequestBuildErrorTestCase( - description="value error", expected_error=ValueError("orders payload has no name") - ), - ], - ids=lambda case: case.description, -) -def test_given_failing_request_builder_when_evaluating_native_then_error_propagates_unchanged( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - test_case: NativeRequestBuildErrorTestCase, -) -> None: - project: CompiledProject = build_project( - name="orders", relative_path="models/orders.sql", sql="SELECT 1", config_values={} - ) - started: list[str] = [] - - def failing_model_payloads(**_: object) -> NoReturn: - raise test_case.expected_error - - def record_custom_start(**_: object) -> None: - started.append("custom") - - def record_native_call(*_: object) -> str: - started.append("native") - return "{}" - - monkeypatch.setattr(native, "_model_payloads", failing_model_payloads) - monkeypatch.setattr(native, "start_custom_rules", record_custom_start) - monkeypatch.setattr(native._native, "parse_rules_parts", record_native_call) - - with pytest.raises(type(test_case.expected_error)) as raised: - native.evaluate_native( - project=project, - config=RulesConfig(cache=RulesCacheConfig(enabled=False)), - project_dir=tmp_path, - catalogue=(), - ) - - assert raised.value is test_case.expected_error - assert started == [] - - -@pytest.mark.parametrize( - "test_case", - [ - NativeRequestEncodeErrorTestCase( - description="integer beyond 64 bits", rejected_value=2**70, expected_message="64-bit" - ), - ], - ids=lambda case: case.description, -) -def test_given_value_the_encoder_rejects_when_evaluating_native_then_rules_error_is_raised( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - test_case: NativeRequestEncodeErrorTestCase, -) -> None: - project: CompiledProject = build_project( - name="orders", relative_path="models/orders.sql", sql="SELECT 1", config_values={} - ) - started: list[str] = [] - - def oversized_model_payloads(**_: object) -> list[dict[str, object]]: - return [{"name": "orders", "priority": test_case.rejected_value}] - - def record_custom_start(**_: object) -> None: - started.append("custom") - - def record_native_call(*_: object) -> str: - started.append("native") - return "{}" - - monkeypatch.setattr(native, "_model_payloads", oversized_model_payloads) - monkeypatch.setattr(native, "start_custom_rules", record_custom_start) - monkeypatch.setattr(native._native, "parse_rules_parts", record_native_call) - - with pytest.raises(native.RulesError, match=test_case.expected_message): - native.evaluate_native( - project=project, - config=RulesConfig(cache=RulesCacheConfig(enabled=False)), - project_dir=tmp_path, - catalogue=(), - ) - - assert started == [] - - -@pytest.mark.parametrize( - "test_case", - [ - NativeRequestEncodeErrorTestCase( - description="model field the native engine rejects", - rejected_value=1, - expected_message="unknown field", - ), - ], - ids=lambda case: case.description, -) -def test_given_payload_the_native_decoder_rejects_when_evaluating_native_then_rules_error_is_raised( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - test_case: NativeRequestEncodeErrorTestCase, -) -> None: - project: CompiledProject = build_project( - name="orders", relative_path="models/orders.sql", sql="SELECT 1", config_values={} - ) - started: list[str] = [] - - def unknown_field_payloads(**_: object) -> list[dict[str, object]]: - return [{"name": "orders", "priority": test_case.rejected_value}] - - def record_custom_start(**_: object) -> None: - started.append("custom") - - monkeypatch.setattr(native, "_model_payloads", unknown_field_payloads) - monkeypatch.setattr(native, "start_custom_rules", record_custom_start) - - with pytest.raises(native.RulesError, match=test_case.expected_message): - native.evaluate_native( - project=project, - config=RulesConfig(cache=RulesCacheConfig(enabled=False)), - project_dir=tmp_path, - catalogue=(), - ) - - assert started == [] - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py index eaa50f35ce..fb8d88879b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py @@ -3,10 +3,8 @@ import pytest +from sqlbuild.compiler.compiled_project.main.declaration_rows import declaration_rows from sqlbuild.compiler.discovery.models import ConstantDeclaration -from sqlbuild.rule_engine._helpers.engine.native import ( - _constant_payload, # noqa: FFL102 - verifies the native protocol payload boundary -) from sqlbuild.sql_values.main.normalize import normalize_sql_value from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( TypedConstantPayloadTestCase, @@ -25,7 +23,7 @@ ], ids=lambda case: case.description, ) -def test_given_typed_constant_when_building_native_payload_then_value_and_type_are_preserved( +def test_given_typed_constant_when_building_native_row_then_value_and_type_are_preserved( test_case: TypedConstantPayloadTestCase, ) -> None: declaration: ConstantDeclaration = ConstantDeclaration( @@ -34,11 +32,14 @@ def test_given_typed_constant_when_building_native_payload_then_value_and_type_a relative_path=Path("constants/rules.sql"), ) - payload: dict[str, object] = _constant_payload(declaration) + _, _, members, value, value_type, render_as = declaration_rows( + enums=(), constants=(declaration,) + )[1][0] - assert payload["value"] == test_case.expected_value - assert payload["value_type"] == test_case.expected_type - assert payload["render_as"] is None + assert members == [] + assert value == test_case.expected_value + assert value_type == test_case.expected_type + assert render_as is None if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index e3271306fc..e805376121 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -5,7 +5,6 @@ import shutil import time from collections.abc import Callable -from dataclasses import replace from operator import attrgetter from pathlib import Path from types import SimpleNamespace @@ -14,7 +13,8 @@ import pytest import sqlbuild._native as native_module -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.host import custom_host_pool @@ -74,7 +74,7 @@ def write_order_rules( ) -def orders_project(*, model_count: int, edits: dict[str, str] | None = None) -> CompiledProject: +def orders_project(*, model_count: int, edits: dict[str, str] | None = None) -> CompiledProjectView: """Return numbered order models whose SQL marks them odd or even, with optional edits.""" sql_by_name: dict[str, str] = { @@ -82,15 +82,17 @@ def orders_project(*, model_count: int, edits: dict[str, str] | None = None) -> for index in range(model_count) } sql_by_name.update(edits or {}) - projects: list[CompiledProject] = [ + projects: list[CompiledProjectView] = [ build_project(name=name, relative_path=f"models/{name}.sql", sql=sql, config_values={}) for name, sql in sql_by_name.items() ] - return replace(projects[0], models=tuple(project.models[0] for project in projects)) + return project_with( + project=projects[0], models=tuple(project.models[0] for project in projects) + ) def evaluate_order_rules( - *, project: CompiledProject, root: Path, cache_enabled: bool = True + *, project: CompiledProjectView, root: Path, cache_enabled: bool = True ) -> CustomRulesOutcome: """Evaluate both order rules through the incremental custom-rule cache.""" @@ -437,7 +439,7 @@ def delay_inputs(*, monkeypatch: pytest.MonkeyPatch, delay_seconds: float) -> No write: Callable[..., None] = custom_host_pool._write_inputs - def delayed_write(*, path: Path, project: CompiledProject, config: RulesConfig) -> None: + def delayed_write(*, path: Path, project: CompiledProjectView, config: RulesConfig) -> None: time.sleep(delay_seconds) write(path=path, project=project, config=config) @@ -447,7 +449,7 @@ def delayed_write(*, path: Path, project: CompiledProject, config: RulesConfig) def fail_inputs(*, monkeypatch: pytest.MonkeyPatch, delay_seconds: float) -> None: """Fail to publish every host project payload after a delay.""" - def failed_write(*, path: Path, project: CompiledProject, config: RulesConfig) -> None: + def failed_write(*, path: Path, project: CompiledProjectView, config: RulesConfig) -> None: del path, project, config time.sleep(delay_seconds) raise OSError("project payload disk is full") @@ -462,7 +464,7 @@ def fail_inputs_without_directory( failures: list[float] = [] - def failed_write(*, path: Path, project: CompiledProject, config: RulesConfig) -> None: + def failed_write(*, path: Path, project: CompiledProjectView, config: RulesConfig) -> None: del project, config time.sleep(delay_seconds) shutil.rmtree(path.parent) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 6738ebdc7d..213a40c2aa 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -5,7 +5,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.host import custom_host, custom_host_pool from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import CUSTOM_RULES_CACHE_FILE @@ -75,7 +75,7 @@ def test_given_partitioned_hosts_when_evaluating_then_findings_and_cache_match_o records_module_state=test_case.records_module_state, detects_duplicates=test_case.detects_duplicates, ) - project: CompiledProject = orders_project(model_count=test_case.model_count) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) cache_path: Path = rules_bulk_cache_path( project_dir=tmp_path, file_name=CUSTOM_RULES_CACHE_FILE ) @@ -129,8 +129,8 @@ def test_given_results_cached_by_split_hosts_when_rerunning_then_only_edited_sub tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostPartitionTestCase ) -> None: write_order_rules(root=tmp_path) - project: CompiledProject = orders_project(model_count=test_case.model_count) - edited: CompiledProject = orders_project( + project: CompiledProjectView = orders_project(model_count=test_case.model_count) + edited: CompiledProjectView = orders_project( model_count=test_case.model_count, edits={"orders_003": "SELECT 2 AS order_id"} ) launches: list[str] = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) @@ -174,7 +174,7 @@ def test_given_failing_rule_on_split_hosts_when_evaluating_then_single_host_erro tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostFailureTestCase ) -> None: write_order_rules(root=tmp_path, failing=test_case.failing_models) - project: CompiledProject = orders_project(model_count=test_case.model_count) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) _ = use_hosts(monkeypatch=monkeypatch, hosts=1) with pytest.raises(RulesError) as single: @@ -246,7 +246,7 @@ def test_given_failing_host_when_others_are_busy_then_they_stop_before_later_inv tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostCancellationTestCase ) -> None: write_order_rules(root=tmp_path, failing=("orders_000",), delay_seconds=test_case.delay_seconds) - project: CompiledProject = orders_project(model_count=test_case.model_count) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) launches: list[str] = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) monkeypatch.setattr(custom_host_pool, "_HOST_TIMEOUT_MILLIS", test_case.timeout_millis) started: float = time.monotonic() @@ -279,7 +279,7 @@ def test_given_cancelled_hosts_when_failing_then_no_process_is_signalled( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostCancellationTestCase ) -> None: write_order_rules(root=tmp_path, failing=("orders_000",), delay_seconds=test_case.delay_seconds) - project: CompiledProject = orders_project(model_count=test_case.model_count) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) _ = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) signals: list[tuple[int, int]] = record_signals(monkeypatch=monkeypatch) @@ -314,7 +314,7 @@ def test_given_hosts_started_before_payload_when_payload_arrives_then_findings_a tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostInputsTestCase ) -> None: write_order_rules(root=tmp_path) - project: CompiledProject = orders_project(model_count=test_case.model_count) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) oracle: CustomRulesOutcome = evaluate_order_rules( project=project, root=tmp_path, cache_enabled=False ) @@ -354,7 +354,7 @@ def test_given_payload_write_failure_when_hosts_wait_then_original_error_is_rais tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostInputsTestCase ) -> None: write_order_rules(root=tmp_path) - project: CompiledProject = orders_project(model_count=test_case.model_count) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) launches: list[str] = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) fail_inputs(monkeypatch=monkeypatch, delay_seconds=test_case.write_delay_seconds) started: float = time.monotonic() @@ -391,7 +391,7 @@ def test_given_input_folder_removed_when_payload_write_fails_then_original_error tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: HostInputsTestCase ) -> None: write_order_rules(root=tmp_path) - project: CompiledProject = orders_project(model_count=test_case.model_count) + project: CompiledProjectView = orders_project(model_count=test_case.model_count) launches: list[str] = use_hosts(monkeypatch=monkeypatch, hosts=test_case.hosts) failures: list[float] = fail_inputs_without_directory( monkeypatch=monkeypatch, delay_seconds=test_case.write_delay_seconds diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/_test_types.py new file mode 100644 index 0000000000..5fc021598a --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/_test_types.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeRowsEncodeErrorTestCase: + """A model config value the native request builder must reject before custom rules start.""" + + description: str + rejected_value: object + expected_message: str + + +@dataclass(frozen=True) +class NativeRowsMemoTestCase: + """Whether each native evaluation reused the memoized response: cold, rebuilt, warm.""" + + description: str + expected_reused: tuple[bool, bool, bool] diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py new file mode 100644 index 0000000000..cb858b6cd6 --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/helpers.py @@ -0,0 +1,44 @@ +from collections.abc import Callable + +import pytest + +from sqlbuild._native import NativeRulesRequest +from sqlbuild.rule_engine._helpers.engine import native +from sqlbuild.rule_engine._helpers.run import native_rows +from sqlbuild.rule_engine.models import NativeRulesEvaluation + + +def record_custom_starts(*, monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Record every custom-rule start instead of starting one.""" + + started: list[str] = [] + + def record_start(**_: object) -> None: + started.append("custom") + + monkeypatch.setattr(native, "start_custom_rules", record_start) + return started + + +def record_reuse(*, monkeypatch: pytest.MonkeyPatch) -> list[bool]: + """Record whether each natively evaluated request was answered from the response memo.""" + + reused: list[bool] = [] + evaluate: Callable[[NativeRulesRequest], NativeRulesEvaluation] = ( + native_rows.evaluate_rules_request + ) + + def recording(request: NativeRulesRequest) -> NativeRulesEvaluation: + evaluation: NativeRulesEvaluation = evaluate(request) + reused.append(evaluation.reused) + return evaluation + + monkeypatch.setattr(native, "evaluate_rules_request", recording) + return reused + + +def rebuild_identity(*, monkeypatch: pytest.MonkeyPatch) -> None: + """Pretend the compiled code changed, as after installing another build.""" + + identity: str = native_rows.compiled_code_identity() + monkeypatch.setattr(native_rows, "compiled_code_identity", lambda: f"{identity}-rebuilt") diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py new file mode 100644 index 0000000000..51c7aff77b --- /dev/null +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/run/test_native_rows.py @@ -0,0 +1,81 @@ +"""Externally visible outcomes of the natively built rules request.""" + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.rule_engine._helpers.engine import native +from sqlbuild.rule_engine.models import RulesCacheConfig, RulesConfig, RulesResult +from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import evaluate_contract_rule +from tests.unit.src.sqlbuild.rule_engine._helpers.run._test_types import ( + NativeRowsEncodeErrorTestCase, + NativeRowsMemoTestCase, +) +from tests.unit.src.sqlbuild.rule_engine._helpers.run.helpers import ( + rebuild_identity, + record_custom_starts, + record_reuse, +) +from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project + + +@pytest.mark.parametrize( + "test_case", + [ + NativeRowsEncodeErrorTestCase( + description="integer beyond 64 bits", rejected_value=2**70, expected_message="64-bit" + ), + ], + ids=lambda case: case.description, +) +def test_given_config_value_the_encoder_rejects_when_building_natively_then_rules_error_is_raised( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + test_case: NativeRowsEncodeErrorTestCase, +) -> None: + started: list[str] = record_custom_starts(monkeypatch=monkeypatch) + project: CompiledProjectView = build_project( + name="orders", + relative_path="models/orders.sql", + sql="SELECT 1", + config_values={"priority": test_case.rejected_value}, + ) + + with pytest.raises(native.RulesError, match=test_case.expected_message): + native.evaluate_native( + project=project, + config=RulesConfig(cache=RulesCacheConfig(enabled=False)), + project_dir=tmp_path, + catalogue=(), + ) + + assert started == [] + + +@pytest.mark.parametrize( + "test_case", + [NativeRowsMemoTestCase("memoized response follows the native build", (False, False, True))], + ids=lambda case: case.description, +) +def test_given_memoized_response_from_another_build_when_evaluating_natively_then_rules_rerun( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + test_case: NativeRowsMemoTestCase, +) -> None: + reused: list[bool] = record_reuse(monkeypatch=monkeypatch) + config_values: dict[str, object] = {"materialized": "table"} + + cold: RulesResult = evaluate_contract_rule( + config_values=config_values, project_dir=tmp_path, cache_enabled=True + ) + rebuild_identity(monkeypatch=monkeypatch) + rebuilt: RulesResult = evaluate_contract_rule( + config_values=config_values, project_dir=tmp_path, cache_enabled=True + ) + warm: RulesResult = evaluate_contract_rule( + config_values=config_values, project_dir=tmp_path, cache_enabled=True + ) + + assert tuple(reused) == test_case.expected_reused + assert rebuilt.findings == warm.findings == cold.findings diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py index 9589d03295..62b22ad19a 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py @@ -1,11 +1,11 @@ from pathlib import Path from sqlbuild.compiler.compile.models import CompiledAudit, CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredAuditBlock, DiscoveredAuditFile -def build_audit(*, name: str, attached_target_name: str | None) -> CompiledAudit: +def build_audit(*, name: str, attached_target_name: str | None) -> CompiledAuditView: relative_path: Path = Path(f"audits/{name}.sql") audit_file: DiscoveredAuditFile = DiscoveredAuditFile( file_path=relative_path, diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py index 22f2bedb52..04ba55c983 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py @@ -1,11 +1,11 @@ from __future__ import annotations -from dataclasses import replace from pathlib import Path import pytest -from sqlbuild.compiler.compile.models import CompiledAudit, CompiledProject +from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledProjectView +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.classes.audit_facts import AuditFacts from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine.classes.audit_facts._test_types import ( @@ -32,17 +32,17 @@ def test_given_attached_and_project_audits_when_querying_model_then_returns_only_attached_audits( test_case: AuditFactsTestCase, ) -> None: - attached: CompiledAudit = build_audit( + attached: CompiledAuditView = build_audit( name=test_case.attached_name, attached_target_name="orders" ) - other_model: CompiledAudit = build_audit( + other_model: CompiledAuditView = build_audit( name=test_case.other_model_name, attached_target_name="customers" ) - project_audit: CompiledAudit = build_audit( + project_audit: CompiledAuditView = build_audit( name=test_case.project_audit_name, attached_target_name=None ) - project: CompiledProject = replace( - build_project( + project: CompiledProjectView = project_with( + project=build_project( name="orders", relative_path="models/orders.sql", sql="SELECT 1 AS order_id", @@ -53,7 +53,7 @@ def test_given_attached_and_project_audits_when_querying_model_then_returns_only facts: AuditFacts = AuditFacts(project=project) - attached_audits: tuple[CompiledAudit, ...] = facts.for_model( + attached_audits: tuple[CompiledAuditView, ...] = facts.for_model( Model(name="orders", path=Path("models/orders.sql"), materialization="view") ) assert tuple(audit.name for audit in attached_audits) == test_case.expected_attached_names diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py index 653bf77f17..d26e3ba301 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py @@ -5,8 +5,9 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest -from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.compile.models import CompiledSqlTest +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.classes.model_test_facts import TestFacts as ModelTestFacts from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( @@ -47,8 +48,8 @@ def test_given_model_and_non_model_tests_when_querying_model_then_returns_each_m direct: CompiledSqlTest = direct_sql_test( mode=SqlTestMode.UDF, name=test_case.direct_test_name, block_index=1 ) - project: CompiledProject = replace( - build_project( + project: CompiledProjectView = project_with( + project=build_project( name=test_case.model_name, relative_path="models/orders.sql", sql="SELECT 1 AS order_id", diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/helpers.py index 387bc9c79d..10ab2663ff 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/project_tree/helpers.py @@ -1,6 +1,6 @@ from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import ProjectTree from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project @@ -24,7 +24,7 @@ def linked_project_tree(tmp_path: Path) -> ProjectTree: (root / "models" / "customers.sql").symlink_to(outside / "customers.sql") (root / "linked_models").symlink_to(root / "models", target_is_directory=True) (root / "outside_models").symlink_to(outside, target_is_directory=True) - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="orders", relative_path="models/orders.sql", sql=ORDERS_SQL, config_values={} ) return ProjectTree(project_dir=root, project=project) diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py index 1fd0d8b87b..4b6084dc9b 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py @@ -1,8 +1,12 @@ -from dataclasses import replace from pathlib import Path -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompiledSource -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSource +from sqlbuild.compiler.compile.types import ( + CompiledProjectView, + CompiledResourceType, + CompiledSourceView, +) +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.spec.contracts.models import SourceEntry @@ -41,7 +45,7 @@ def _source_file(*, tmp_path: Path, names: tuple[str, ...], contents: str) -> Di def _views( *, tmp_path: Path, files: tuple[DiscoveredSourceFile, ...], names: tuple[str, ...] ) -> RuleFactViews: - sources: tuple[CompiledSource, ...] = tuple( + sources: tuple[CompiledSourceView, ...] = tuple( CompiledSource( key=CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=name), deps=(), @@ -51,8 +55,8 @@ def _views( ) for name, declared in zip(names, files, strict=True) ) - project: CompiledProject = replace( - build_project( + project: CompiledProjectView = project_with( + project=build_project( name="orders", relative_path="models/orders.sql", sql="SELECT 1", config_values={} ), sources=sources, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py index e1e16fa077..07a697acca 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py @@ -10,7 +10,11 @@ CompileModelConfig, CompileSqlReference, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import ( + CompiledModelView, + CompiledProjectView, + CompiledResourceType, +) from sqlbuild.compiler.discovery.models import EnumDeclaration, EnumMember _ENUM_DECLARATION: EnumDeclaration = EnumDeclaration( @@ -31,8 +35,8 @@ def build_project( references: tuple[CompileSqlReference, ...] = (), authored_sql: str | None = None, enum_columns: tuple[str, ...] = (), -) -> CompiledProject: - model: CompiledModel = CompiledModel( +) -> CompiledProjectView: + model: CompiledModelView = CompiledModel( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name), deps=(), name=name, diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py index 5e2d6ed6cb..df0e6a0334 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py @@ -5,7 +5,9 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record +from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import RuleExemption, RuleIgnore, RulesConfig, RulesResult @@ -28,7 +30,7 @@ def test_given_unchanged_builtin_inputs_when_evaluating_twice_then_second_run_hi tmp_path: Path, test_case: PolicyBehaviorTestCase, ) -> None: - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="commerce__mart__orders", relative_path="models/mart/commerce__mart__orders.sql", sql="SELECT * FROM orders", @@ -63,20 +65,20 @@ def test_given_authored_sql_changes_when_compiled_sql_is_same_then_cache_is_inva test_case: PolicyBehaviorTestCase, ) -> None: compiled_sql: str = "WITH final AS (SELECT 1 AS id) SELECT id FROM final WHERE id > 7" - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="commerce__mart__orders", relative_path="models/mart/commerce__mart__orders.sql", sql=compiled_sql, config_values={}, ) constant_source: str = compiled_sql.replace("7", '@const("minimum_id")') - first_project: CompiledProject = replace( - project, - models=(replace(project.models[0], authored_sql=constant_source),), + first_project: CompiledProjectView = project_with( + project=project, + models=(replace(compiled_model_record(project.models[0]), authored_sql=constant_source),), ) - bare_project: CompiledProject = replace( - project, - models=(replace(project.models[0], authored_sql=compiled_sql),), + bare_project: CompiledProjectView = project_with( + project=project, + models=(replace(compiled_model_record(project.models[0]), authored_sql=compiled_sql),), ) config: RulesConfig = RulesConfig(select=("SQBRDECLARATION102",)) _ = evaluate(project=first_project, config=config, project_dir=tmp_path) @@ -106,7 +108,7 @@ def test_given_exact_exception_when_fault_exists_then_suppresses_fault( target: Path = tmp_path / relative_path target.parent.mkdir(parents=True) target.write_text("SELECT * FROM orders\n", encoding="utf-8") - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="commerce__mart__orders", relative_path=relative_path, sql="SELECT * FROM orders", @@ -146,7 +148,7 @@ def test_given_stale_exact_exception_when_evaluating_then_raises_error( target: Path = tmp_path / relative_path target.parent.mkdir(parents=True) target.write_text("WITH final AS (SELECT 1 AS id) SELECT id FROM final\n", encoding="utf-8") - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="commerce__mart__orders", relative_path=relative_path, sql="WITH final AS (SELECT 1 AS id) SELECT id FROM final", @@ -178,7 +180,7 @@ def test_given_scoped_ignore_when_fault_exists_then_suppresses_fault( test_case: PolicyBehaviorTestCase, ) -> None: relative_path = "models/legacy/commerce__mart__orders.sql" - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="commerce__mart__orders", relative_path=relative_path, sql="SELECT * FROM orders", diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py index bfce6b065f..c728249c29 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate_rule/test_evaluate_rule.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.constants import MIN_CUSTOM_RULE_TEST_CASES from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate @@ -84,7 +84,7 @@ def test_given_same_custom_rule_when_using_public_and_native_paths_then_faults_h ) -> None: rule_case: RuleCase = test_case.rule_case public_result: RuleResult = evaluate_rule(rule=required_domain, test_case=rule_case) - project: CompiledProject = build_project( + project: CompiledProjectView = build_project( name="commerce__mart__orders", relative_path=rule_case.path, sql="WITH final AS (SELECT 1 AS id) SELECT id FROM final", diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py index caa388cd75..e2b44d8e12 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py @@ -8,6 +8,7 @@ LoaderDestinationParts, LocalConfig, ProjectConfig, + TargetConfig, TargetWarehousesConfig, ) from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy @@ -91,3 +92,12 @@ class AmbiguousTargetConnectionTestCase: project_config: ProjectConfig local_config: LocalConfig expected_error_fragment: str + + +@dataclass(frozen=True) +class FullTargetResolutionTestCase: + """A project target setting every field, merged with an empty local target.""" + + description: str + project_target: TargetConfig + expected_target: TargetConfig diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py index 00ee47ce0c..c8a5971a79 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py @@ -6,23 +6,84 @@ from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, + ClonePolicy, ExecutionLimitsConfig, LocalConfig, LocalTargetConfig, ProjectConfig, TargetConfig, + TargetDiffConfig, TargetWarehousesConfig, ) -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy +from sqlbuild.spec.contracts.types import ( + MissingMigrationOriginPolicy, + RetentionDecreasePolicy, + TableType, + TableTypeDowngradePolicy, +) from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( AmbiguousTargetConnectionTestCase, ExecutionLimitsResolutionTestCase, + FullTargetResolutionTestCase, MissingOriginPolicyResolutionTestCase, TargetConnectionDefaultTestCase, TargetRetentionResolutionTestCase, TargetWarehousesResolutionTestCase, ) +_FULL_TARGET: TargetConfig = TargetConfig( + connection={"database": "orders.duckdb"}, + connection_name="warehouse", + vars={"region": "eu"}, + database="orders", + schema="sales", + loader_schema="raw", + defer_sources_to="prod", + defer_clone_from="prod", + clone=ClonePolicy(allow_as_clone_origin=True, allow_as_clone_destination=True), + compile_cache=False, + time_travel_retention=AuthoredTimeTravelRetention(desired_days=3), + time_travel_retention_by_materialization={"table": AuthoredTimeTravelRetention(desired_days=5)}, + owns_time_travel_retention_namespace=True, + default_table_type=TableType.TRANSIENT, + table_type_downgrade=TableTypeDowngradePolicy.DENY, + time_travel_retention_decrease=RetentionDecreasePolicy.REQUIRE_CONFIRMATION, + missing_migration_origin=MissingMigrationOriginPolicy.DENY, + execution_limits=ExecutionLimitsConfig( + max_models=4, max_duration="1h", max_duration_seconds=3600, remediation="stop" + ), + diff=TargetDiffConfig(max_full_rows=10), + warehouses=TargetWarehousesConfig(build="build_wh", query="query_wh"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + FullTargetResolutionTestCase( + description="every project target field survives an empty local target", + project_target=_FULL_TARGET, + expected_target=_FULL_TARGET, + ) + ], + ids=lambda case: case.description, +) +def test_given_full_project_target_when_resolving_with_empty_local_then_every_field_is_kept( + test_case: FullTargetResolutionTestCase, +) -> None: + resolved: TargetConfig = resolve_target_config( + project_config=ProjectConfig( + name="demo", + adapter="snowflake", + connections={"warehouse": {}}, + targets={"dev": test_case.project_target}, + ), + local_config=LocalConfig(targets={"dev": LocalTargetConfig()}), + target_name="dev", + ) + + assert resolved == test_case.expected_target + @pytest.mark.parametrize( "test_case", diff --git a/tests/unit/src/sqlbuild/sql_values/test_normalization.py b/tests/unit/src/sqlbuild/sql_values/test_normalization.py index 9694b733d7..f2aaf11cec 100644 --- a/tests/unit/src/sqlbuild/sql_values/test_normalization.py +++ b/tests/unit/src/sqlbuild/sql_values/test_normalization.py @@ -5,7 +5,6 @@ import pytest -from sqlbuild.sql_values._helpers.normalization import sql_value_identity from sqlbuild.sql_values.exceptions import SqlValueValidationError from sqlbuild.sql_values.main.normalize import normalize_sql_value from sqlbuild.sql_values.main.validate_rendered_size import validate_rendered_sql_value_size @@ -79,12 +78,8 @@ def test_given_lists_sets_and_objects_when_normalizing_then_canonicalizes_unorde list_items: tuple[SqlValue, ...] = cast(tuple[SqlValue, ...], list_value.value) assert tuple(item.value for item in list_items) == test_case.expected_list_values - assert ( - sql_value_identity(value=first_set) == sql_value_identity(value=second_set) - ) is test_case.expected_identities_equal - assert ( - sql_value_identity(value=first_object) == sql_value_identity(value=second_object) - ) is test_case.expected_identities_equal + assert (first_set == second_set) is test_case.expected_identities_equal + assert (first_object == second_object) is test_case.expected_identities_equal @pytest.mark.parametrize( @@ -126,9 +121,7 @@ def test_given_python_equal_cross_type_values_when_identifying_then_identity_rem boolean: SqlValue = normalize_sql_value(raw_value=True, context="constant 'value'") integer: SqlValue = normalize_sql_value(raw_value=1, context="constant 'value'") - assert ( - sql_value_identity(value=boolean) == sql_value_identity(value=integer) - ) is test_case.expected_identities_equal + assert (boolean == integer) is test_case.expected_identities_equal @pytest.mark.parametrize( diff --git a/uv.lock b/uv.lock index bf8b9ccf02..7f3cfb3a50 100644 --- a/uv.lock +++ b/uv.lock @@ -4147,6 +4147,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "pyyaml" }, { name = "rich" }, + { name = "sqlbuild-cli" }, ] [package.optional-dependencies] @@ -4226,6 +4227,7 @@ requires-dist = [ { name = "rich", specifier = ">=15.0.0" }, { name = "rivers", marker = "extra == 'rivers'", specifier = ">=0.1.4" }, { name = "snowflake-connector-python", marker = "extra == 'snowflake'", specifier = ">=4.7.1,<5" }, + { name = "sqlbuild-cli", directory = "crates/sqlbuild-cli" }, ] provides-extras = ["dbt", "snowflake", "bigquery", "databricks", "dagster", "postgres", "sqlserver", "ingestr", "dlt", "rivers"] @@ -4253,6 +4255,10 @@ dev = [ { name = "ty", specifier = ">=0.0.1a19" }, ] +[[package]] +name = "sqlbuild-cli" +source = { directory = "crates/sqlbuild-cli" } + [[package]] name = "sqlglot" version = "30.11.0" diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 526201d0a8..a08b7cbc86 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -111,7 +111,7 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as `settings_customise_sources` or command-line parsing. The reason is logged at debug level. SQLBuild keeps one stored result per project and target under -`target/cache/compiler-native-v1/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is +`target/cache/compiler-native-v1/project-reuse-v2/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the report is ready. When it has to hash a lot of data or the project has very many files, compile prints `Recording compile for reuse...` and a completion line to stderr before the report. @@ -120,7 +120,8 @@ prints `Recording compile for reuse...` and a completion line to stderr before t Any change runs the whole compile again: every model, test, and audit is rendered again, and contracts and semantic validation run for the whole project. No render is carried over from the -previous compile; rendering runs natively and does not run unchanged macro calls again. A few +previous compile; rendering runs natively and does not run unchanged macro calls again. Every +model's SQL references are scanned again too; no reference scan is kept between compiles. A few caches under `target/cache/` shorten the rest: - An unchanged model takes its column analysis from `model-analyses.bin`. Entries are keyed by